From 5b3406551b2385abe4efa1b6320243d64de54030 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 29 Sep 2022 11:47:27 +0200
Subject: [PATCH 001/922] added api-direct-upload option for storage
 configurations

---
 doc/release-notes/4.20-release-notes.md               | 11 +++++++++--
 doc/sphinx-guides/source/installation/config.rst      |  1 +
 .../harvard/iq/dataverse/dataaccess/StorageIO.java    |  3 ++-
 .../java/edu/harvard/iq/dataverse/util/FileUtil.java  |  3 ++-
 .../edu/harvard/iq/dataverse/util/SystemConfig.java   |  3 ++-
 5 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/doc/release-notes/4.20-release-notes.md b/doc/release-notes/4.20-release-notes.md
index e29953db101..8044047134f 100644
--- a/doc/release-notes/4.20-release-notes.md
+++ b/doc/release-notes/4.20-release-notes.md
@@ -90,10 +90,16 @@ Also note that the :MaxFileUploadSizeInBytes property has a new option to provid
 
 ### Direct S3 Upload Changes
 
-Direct upload to S3 is enabled per store by one new jvm option:
+Direct upload to S3 in UI and API is enabled per store by one new jvm option:
 
     ./asadmin create-jvm-options "\-Ddataverse.files.<id>.upload-redirect=true"
-    
+
+That option makes direct upload by default in UI, in the API you can use either: direct or through Dataverse upload. Direct upload to S3 in API only is enabled per store by this new jvm option:
+
+    ./asadmin create-jvm-options "\-Ddataverse.files.<id>.api-direct-upload=true"
+
+That option leaves through Dataverse upload by default in UI, but makes both: through Dataverse and direct uploads possible via API.
+
 The existing :MaxFileUploadSizeInBytes property and ```dataverse.files.<id>.url-expiration-minutes``` jvm option for the same store also apply to direct upload.
 
 Direct upload via the Dataverse web interface is transparent to the user and handled automatically by the browser. Some minor differences in file upload exist: directly uploaded files are not unzipped and Dataverse does not scan their content to help in assigning a MIME type. Ingest of tabular files and metadata extraction from FITS files will occur, but can be turned off for files above a specified size limit through the new dataverse.files.<id>.ingestsizelimit jvm option.
@@ -127,6 +133,7 @@ We made changes to the JSON Export in this release (Issue 6650, PR #6669). If yo
 
 - The JVM option dataverse.files.file.directory=<your directory> controls where temporary files are stored (in the /temp subdir of the defined directory), independent of the location of any 'file' store defined above.
 - The JVM option dataverse.files.<id>.upload-redirect enables direct upload of files added to a dataset to the S3 bucket. (S3 stores only!)
+- The JVM option dataverse.files.<id>.api-direct-upload enables direct upload of files added to a dataset to any storage. (Through API only and when the uploading tool has direct access to the storage used; i.e., uplad the file first and register it through API!)
 - The JVM option dataverse.files.<id>.MaxFileUploadSizeInBytes controls the maximum size of file uploads allowed for the given file store. 
 - The JVM option dataverse.files.<id>.ingestsizelimit controls the maximum size of files for which ingest will be attempted, for the given file store. 
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f2de9d5702f..2b605ae8945 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -546,6 +546,7 @@ List of S3 Storage Options
     dataverse.files.<id>.bucket-name             <?>                 The bucket name. See above.                                                 (none)
     dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download or proxy through Dataverse.                          ``false``
     dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset  to the S3 store.          ``false``
+    dataverse.files.<id>.api-direct-upload       ``true``/``false``  Enable direct upload of files added to a dataset through API only.          ``false``
     dataverse.files.<id>.ingestsizelimit         <size in bytes>     Maximum size of directupload files that should be ingested                  (none)
     dataverse.files.<id>.url-expiration-minutes  <?>                 If direct uploads/downloads: time until links expire. Optional.             60
     dataverse.files.<id>.min-part-size           <?>                 Multipart direct uploads will occur for files larger than this. Optional.   ``1024**3``
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index 90e4a54dbe8..8e2dd9fa961 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -605,7 +605,8 @@ public static String getDriverPrefix(String driverId) {
     }
     
     public static boolean isDirectUploadEnabled(String driverId) {
-        return Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect"));
+        return Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".api-direct-upload"));
     }
     
     //Check that storageIdentifier is consistent with store's config
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 339de904f9e..0a41da4f7dd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1748,7 +1748,8 @@ public static boolean isPackageFile(DataFile dataFile) {
     
     public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	String driverId = dataset.getEffectiveStorageDriverId();
-    	boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect");
+    	boolean directEnabled = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".api-direct-upload"));
     	//Should only be requested when it is allowed, but we'll log a warning otherwise
     	if(!directEnabled) {
     		logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId());
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 7abd0d02065..4553a71a1d2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -1112,7 +1112,8 @@ public boolean isDatafileValidationOnPublishEnabled() {
     }
 
 	public boolean directUploadEnabled(DvObjectContainer container) {
-    	return Boolean.getBoolean("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect");
+        // this method is used in UI only, therfore "dataverse.files." + driverId + ".api-direct-upload" is not used here
+    	return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
 	}
 	
 	public String getDataCiteRestApiUrlString() {

From 5db560e999454a0b974215c6d9bc8373d4595fc0 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 29 Sep 2022 13:43:33 +0200
Subject: [PATCH 002/922] improvements in the documentation

---
 doc/release-notes/4.20-release-notes.md          | 8 ++++----
 doc/sphinx-guides/source/installation/config.rst | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/release-notes/4.20-release-notes.md b/doc/release-notes/4.20-release-notes.md
index 8044047134f..79037d8cd8c 100644
--- a/doc/release-notes/4.20-release-notes.md
+++ b/doc/release-notes/4.20-release-notes.md
@@ -94,11 +94,11 @@ Direct upload to S3 in UI and API is enabled per store by one new jvm option:
 
     ./asadmin create-jvm-options "\-Ddataverse.files.<id>.upload-redirect=true"
 
-That option makes direct upload by default in UI, in the API you can use either: direct or through Dataverse upload. Direct upload to S3 in API only is enabled per store by this new jvm option:
+This option makes direct upload the default in the UI. In the API, you can use either: direct upload or upload via Dataverse upload. Direct upload to S3 in API only is enabled per store by this new jvm option:
 
     ./asadmin create-jvm-options "\-Ddataverse.files.<id>.api-direct-upload=true"
 
-That option leaves through Dataverse upload by default in UI, but makes both: through Dataverse and direct uploads possible via API.
+That option leaves via Dataverse upload by default in UI, but makes both: uploads via Dataverse and direct uploads possible via API.
 
 The existing :MaxFileUploadSizeInBytes property and ```dataverse.files.<id>.url-expiration-minutes``` jvm option for the same store also apply to direct upload.
 
@@ -132,8 +132,8 @@ We made changes to the JSON Export in this release (Issue 6650, PR #6669). If yo
 ## New JVM Options for file storage drivers
 
 - The JVM option dataverse.files.file.directory=<your directory> controls where temporary files are stored (in the /temp subdir of the defined directory), independent of the location of any 'file' store defined above.
-- The JVM option dataverse.files.<id>.upload-redirect enables direct upload of files added to a dataset to the S3 bucket. (S3 stores only!)
-- The JVM option dataverse.files.<id>.api-direct-upload enables direct upload of files added to a dataset to any storage. (Through API only and when the uploading tool has direct access to the storage used; i.e., uplad the file first and register it through API!)
+- The JVM option dataverse.files.<id>.upload-redirect enables direct upload of files added to a dataset in the S3 bucket. (S3 stores only!)
+- The JVM option dataverse.files.<id>.api-direct-upload enables direct upload of files added to a dataset in any storage. (Via API only and when the uploading tool has direct access to the relevant storage used; i.e., upload the file first and register it via API!)
 - The JVM option dataverse.files.<id>.MaxFileUploadSizeInBytes controls the maximum size of file uploads allowed for the given file store. 
 - The JVM option dataverse.files.<id>.ingestsizelimit controls the maximum size of files for which ingest will be attempted, for the given file store. 
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 2b605ae8945..3245aeccfaf 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -545,8 +545,8 @@ List of S3 Storage Options
     dataverse.files.<id>.label                   <?>                 **Required** label to be shown in the UI for this storage                   (none)
     dataverse.files.<id>.bucket-name             <?>                 The bucket name. See above.                                                 (none)
     dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download or proxy through Dataverse.                          ``false``
-    dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset  to the S3 store.          ``false``
-    dataverse.files.<id>.api-direct-upload       ``true``/``false``  Enable direct upload of files added to a dataset through API only.          ``false``
+    dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset in the S3 store.           ``false``
+    dataverse.files.<id>.api-direct-upload       ``true``/``false``  Enable direct upload of files added to a dataset via API only.              ``false``
     dataverse.files.<id>.ingestsizelimit         <size in bytes>     Maximum size of directupload files that should be ingested                  (none)
     dataverse.files.<id>.url-expiration-minutes  <?>                 If direct uploads/downloads: time until links expire. Optional.             60
     dataverse.files.<id>.min-part-size           <?>                 Multipart direct uploads will occur for files larger than this. Optional.   ``1024**3``

From c7cfb533b261b7e36afd78b2061d48d2f55c4736 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Mon, 3 Oct 2022 01:17:42 +0200
Subject: [PATCH 003/922] Remove unused GPL-licensed code

For unknown reasons, in 2009 several files from the JDK were copied into
the Dataverse codebase, instead of referenced.
It appears that these classes weren't really used.
---
 .../spi/FileMetadataExtractorSpi.java         |  18 -
 .../ingest/plugin/spi/DigraphNode.java        | 188 ----
 .../plugin/spi/IngestServiceProvider.java     |  25 +-
 .../plugin/spi/PartiallyOrderedSet.java       | 241 -----
 .../plugin/spi/RegisterableService.java       |  90 --
 .../ingest/plugin/spi/ServiceRegistry.java    | 861 ------------------
 6 files changed, 1 insertion(+), 1422 deletions(-)
 delete mode 100644 src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/DigraphNode.java
 delete mode 100644 src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/PartiallyOrderedSet.java
 delete mode 100644 src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/RegisterableService.java
 delete mode 100644 src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/ServiceRegistry.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/spi/FileMetadataExtractorSpi.java b/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/spi/FileMetadataExtractorSpi.java
index ab8f610cb06..a30dfafe67f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/spi/FileMetadataExtractorSpi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/metadataextraction/spi/FileMetadataExtractorSpi.java
@@ -10,9 +10,7 @@
 import java.util.logging.*;
 import java.io.*;
 
-import edu.harvard.iq.dataverse.ingest.plugin.spi.RegisterableService;
 import edu.harvard.iq.dataverse.ingest.plugin.spi.IngestServiceProvider;
-import edu.harvard.iq.dataverse.ingest.plugin.spi.ServiceRegistry;
 import java.nio.MappedByteBuffer;
 import java.util.Locale;
 
@@ -44,22 +42,6 @@ public FileMetadataExtractorSpi(String vendorName, String version) {
         this.version = version;
     }
 
-    public void onRegistration(ServiceRegistry registry,
-                               Class<?> category) {}
-                               
-    
-    public void onDeregistration(ServiceRegistry registry,
-                                 Class<?> category) {}
-    
-    public String getVersion() {
-        return version;
-    }
-
-    public String getVendorName() {
-        return vendorName;
-    }
-
-    
     public abstract String getDescription(Locale locale);
     
     protected String[] names = null;
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/DigraphNode.java b/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/DigraphNode.java
deleted file mode 100644
index 4db48b5c06a..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/DigraphNode.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
-   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-   Dataverse Network - A web application to share, preserve and analyze research data.
-   Developed at the Institute for Quantitative Social Science, Harvard University.
-   Version 3.0.
-*/
-
-package edu.harvard.iq.dataverse.ingest.plugin.spi;
-
-// This file was Taken out from openjdk-6-src-b16-24_apr_2009.tar.gz
-// http://download.java.net/openjdk/jdk6/promoted/b16/openjdk-6-src-b16-24_apr_2009.tar.gz
-// downloaded: 2009-05-07
-
-
-/*
- * Copyright 2000 Sun Microsystems, Inc.  All Rights Reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Sun designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Sun in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
- */
-
-//package javax.imageio.spi;
-
-import java.io.Serializable;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-/**
- * A node in a directed graph.  In addition to an arbitrary
- * <code>Object</code> containing user data associated with the node,
- * each node maintains a <code>Set</code>s of nodes which are pointed
- * to by the current node (available from <code>getOutNodes</code>).
- * The in-degree of the node (that is, number of nodes that point to
- * the current node) may be queried.
- *
- */
-class DigraphNode implements Cloneable, Serializable {
-
-    /** The data associated with this node. */
-    protected Object data;
-
-    /**
-     * A <code>Set</code> of neighboring nodes pointed to by this
-     * node.
-     */
-    protected Set outNodes = new HashSet();
-
-    /** The in-degree of the node. */
-    protected int inDegree = 0;
-
-    /**
-     * A <code>Set</code> of neighboring nodes that point to this
-     * node.
-     */
-    private Set inNodes = new HashSet();
-
-    public DigraphNode(Object data) {
-        this.data = data;
-    }
-
-    /** Returns the <code>Object</code> referenced by this node. */
-    public Object getData() {
-        return data;
-    }
-
-    /**
-     * Returns an <code>Iterator</code> containing the nodes pointed
-     * to by this node.
-     */
-    public Iterator getOutNodes() {
-        return outNodes.iterator();
-    }
-
-    /**
-     * Adds a directed edge to the graph.  The outNodes list of this
-     * node is updated and the in-degree of the other node is incremented.
-     *
-     * @param node a <code>DigraphNode</code>.
-     *
-     * @return <code>true</code> if the node was not previously the
-     * target of an edge.
-     */
-    public boolean addEdge(DigraphNode node) {
-        if (outNodes.contains(node)) {
-            return false;
-        }
-
-        outNodes.add(node);
-        node.inNodes.add(this);
-        node.incrementInDegree();
-        return true;
-    }
-
-    /**
-     * Returns <code>true</code> if an edge exists between this node
-     * and the given node.
-     *
-     * @param node a <code>DigraphNode</code>.
-     *
-     * @return <code>true</code> if the node is the target of an edge.
-     */
-    public boolean hasEdge(DigraphNode node) {
-        return outNodes.contains(node);
-    }
-
-    /**
-     * Removes a directed edge from the graph.  The outNodes list of this
-     * node is updated and the in-degree of the other node is decremented.
-     *
-     * @return <code>true</code> if the node was previously the target
-     * of an edge.
-     */
-    public boolean removeEdge(DigraphNode node) {
-        if (!outNodes.contains(node)) {
-            return false;
-        }
-
-        outNodes.remove(node);
-        node.inNodes.remove(this);
-        node.decrementInDegree();
-        return true;
-    }
-
-    /**
-     * Removes this node from the graph, updating neighboring nodes
-     * appropriately.
-     */
-    public void dispose() {
-        Object[] inNodesArray = inNodes.toArray();
-        for(int i=0; i<inNodesArray.length; i++) {
-            DigraphNode node = (DigraphNode) inNodesArray[i];
-            node.removeEdge(this);
-        }
-
-        Object[] outNodesArray = outNodes.toArray();
-        for(int i=0; i<outNodesArray.length; i++) {
-            DigraphNode node = (DigraphNode) outNodesArray[i];
-            removeEdge(node);
-        }
-    }
-
-    /** Returns the in-degree of this node. */
-    public int getInDegree() {
-        return inDegree;
-    }
-
-    /** Increments the in-degree of this node. */
-    private void incrementInDegree() {
-        ++inDegree;
-    }
-
-    /** Decrements the in-degree of this node. */
-    private void decrementInDegree() {
-        --inDegree;
-    }
-}
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/IngestServiceProvider.java b/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/IngestServiceProvider.java
index c327ecb2da1..c792156d648 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/IngestServiceProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/IngestServiceProvider.java
@@ -29,7 +29,7 @@
  * 
  * @author akio sone at UNC-Odum
  */
-public abstract class IngestServiceProvider  implements RegisterableService {
+public abstract class IngestServiceProvider  {
 
     /**
      * The name of the vendor that is responsible for coding this
@@ -62,29 +62,6 @@ public IngestServiceProvider(String vendorName, String version) {
     public IngestServiceProvider() {
     }
 
-    /**
-     * A callback to be called exactly once after this Spi class
-     * has been instantiated and registered in a 
-     * <code>ServiceRegistry</code>.
-     *
-     * @param registry the ServiceRegistry instance.
-     * @param category a <code>Class</code>  object that indicatges
-     * its registry category under which this object has been registered.
-     * category.
-     */
-    public void onRegistration(ServiceRegistry registry,
-                               Class<?> category) {}
-                               
-    /**
-     * A callback whenever this Spi class is deregistered from
-     * a <code>ServiceRegistry</code>.
-     *
-     * @param registry the ServiceRegistry instance.
-     * @param category a <code>Class</code> object that indicatges
-     * its registry category from which this object is being de-registered.
-     */
-    public void onDeregistration(ServiceRegistry registry,
-                                 Class<?> category) {}
     /**
      * Gets the value of the version field.
      *
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/PartiallyOrderedSet.java b/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/PartiallyOrderedSet.java
deleted file mode 100644
index 87f4f57cdb6..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/PartiallyOrderedSet.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
-   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-   Dataverse Network - A web application to share, preserve and analyze research data.
-   Developed at the Institute for Quantitative Social Science, Harvard University.
-   Version 3.0.
-*/
-
-package edu.harvard.iq.dataverse.ingest.plugin.spi;
-
-// This file was Taken out from openjdk-6-src-b16-24_apr_2009.tar.gz
-// http://download.java.net/openjdk/jdk6/promoted/b16/openjdk-6-src-b16-24_apr_2009.tar.gz
-// downloaded: 2009-05-07
-
-
-/*
- * Copyright 2000 Sun Microsystems, Inc.  All Rights Reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Sun designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Sun in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
- */
-
-//package javax.imageio.spi;
-
-import java.util.AbstractSet;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * A set of <code>Object</code>s with pairwise orderings between them.
- * The <code>iterator</code> method provides the elements in
- * topologically sorted order.  Elements participating in a cycle
- * are not returned.
- *
- * Unlike the <code>SortedSet</code> and <code>SortedMap</code>
- * interfaces, which require their elements to implement the
- * <code>Comparable</code> interface, this class receives ordering
- * information via its <code>setOrdering</code> and
- * <code>unsetPreference</code> methods.  This difference is due to
- * the fact that the relevant ordering between elements is unlikely to
- * be inherent in the elements themselves; rather, it is set
- * dynamically accoring to application policy.  For example, in a
- * service provider registry situation, an application might allow the
- * user to set a preference order for service provider objects
- * supplied by a trusted vendor over those supplied by another.
- *
- */
-class PartiallyOrderedSet extends AbstractSet {
-
-    // The topological sort (roughly) follows the algorithm described in
-    // Horowitz and Sahni, _Fundamentals of Data Structures_ (1976),
-    // p. 315.
-
-    // Maps Objects to DigraphNodes that contain them
-    private Map poNodes = new HashMap();
-
-    // The set of Objects
-    private Set nodes = poNodes.keySet();
-
-    /**
-     * Constructs a <code>PartiallyOrderedSet</code>.
-     */
-    public PartiallyOrderedSet() {}
-
-    public int size() {
-        return nodes.size();
-    }
-
-    public boolean contains(Object o) {
-        return nodes.contains(o);
-    }
-
-    /**
-     * Returns an iterator over the elements contained in this
-     * collection, with an ordering that respects the orderings set
-     * by the <code>setOrdering</code> method.
-     */
-    public Iterator iterator() {
-        return new PartialOrderIterator(poNodes.values().iterator());
-    }
-
-    /**
-     * Adds an <code>Object</code> to this
-     * <code>PartiallyOrderedSet</code>.
-     */
-    public boolean add(Object o) {
-        if (nodes.contains(o)) {
-            return false;
-        }
-
-        DigraphNode node = new DigraphNode(o);
-        poNodes.put(o, node);
-        return true;
-    }
-
-    /**
-     * Removes an <code>Object</code> from this
-     * <code>PartiallyOrderedSet</code>.
-     */
-    public boolean remove(Object o) {
-        DigraphNode node = (DigraphNode)poNodes.get(o);
-        if (node == null) {
-            return false;
-        }
-
-        poNodes.remove(o);
-        node.dispose();
-        return true;
-    }
-
-    public void clear() {
-        poNodes.clear();
-    }
-
-    /**
-     * Sets an ordering between two nodes.  When an iterator is
-     * requested, the first node will appear earlier in the
-     * sequence than the second node.  If a prior ordering existed
-     * between the nodes in the opposite order, it is removed.
-     *
-     * @return <code>true</code> if no prior ordering existed
-     * between the nodes, <code>false</code>otherwise.
-     */
-    public boolean setOrdering(Object first, Object second) {
-        DigraphNode firstPONode =
-            (DigraphNode)poNodes.get(first);
-        DigraphNode secondPONode =
-            (DigraphNode)poNodes.get(second);
-
-        secondPONode.removeEdge(firstPONode);
-        return firstPONode.addEdge(secondPONode);
-    }
-
-    /**
-     * Removes any ordering between two nodes.
-     *
-     * @return true if a prior prefence existed between the nodes.
-     */
-    public boolean unsetOrdering(Object first, Object second) {
-        DigraphNode firstPONode =
-            (DigraphNode)poNodes.get(first);
-        DigraphNode secondPONode =
-            (DigraphNode)poNodes.get(second);
-
-        return firstPONode.removeEdge(secondPONode) ||
-            secondPONode.removeEdge(firstPONode);
-    }
-
-    /**
-     * Returns <code>true</code> if an ordering exists between two
-     * nodes.
-     */
-    public boolean hasOrdering(Object preferred, Object other) {
-        DigraphNode preferredPONode =
-            (DigraphNode)poNodes.get(preferred);
-        DigraphNode otherPONode =
-            (DigraphNode)poNodes.get(other);
-
-        return preferredPONode.hasEdge(otherPONode);
-    }
-}
-
-class PartialOrderIterator implements Iterator {
-
-    LinkedList zeroList = new LinkedList();
-    Map inDegrees = new HashMap(); // DigraphNode -> Integer
-
-    public PartialOrderIterator(Iterator iter) {
-        // Initialize scratch in-degree values, zero list
-        while (iter.hasNext()) {
-            DigraphNode node = (DigraphNode)iter.next();
-            int inDegree = node.getInDegree();
-            inDegrees.put(node, new Integer(inDegree));
-
-            // Add nodes with zero in-degree to the zero list
-            if (inDegree == 0) {
-                zeroList.add(node);
-            }
-        }
-    }
-
-    public boolean hasNext() {
-        return !zeroList.isEmpty();
-    }
-
-    public Object next() {
-        DigraphNode first = (DigraphNode)zeroList.removeFirst();
-
-        // For each out node of the output node, decrement its in-degree
-        Iterator outNodes = first.getOutNodes();
-        while (outNodes.hasNext()) {
-            DigraphNode node = (DigraphNode)outNodes.next();
-            int inDegree = ((Integer)inDegrees.get(node)).intValue() - 1;
-            inDegrees.put(node, new Integer(inDegree));
-
-            // If the in-degree has fallen to 0, place the node on the list
-            if (inDegree == 0) {
-                zeroList.add(node);
-            }
-        }
-
-        return first.getData();
-    }
-
-    public void remove() {
-        throw new UnsupportedOperationException();
-    }
-}
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/RegisterableService.java b/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/RegisterableService.java
deleted file mode 100644
index d3609b1e4b9..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/RegisterableService.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
-   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-   Dataverse Network - A web application to share, preserve and analyze research data.
-   Developed at the Institute for Quantitative Social Science, Harvard University.
-   Version 3.0.
-*/
-
-package edu.harvard.iq.dataverse.ingest.plugin.spi;
-
-// This file was Taken out from openjdk-6-src-b16-24_apr_2009.tar.gz
-// http://download.java.net/openjdk/jdk6/promoted/b16/openjdk-6-src-b16-24_apr_2009.tar.gz
-// downloaded: 2009-05-07
-
-
-/*
- * Copyright 2000-2004 Sun Microsystems, Inc.  All Rights Reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Sun designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Sun in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
- */
-
-
-/**
- * An optional interface that may be provided by service provider
- * objects that will be registered with a
- * <code>ServiceRegistry</code>.  If this interface is present,
- * notification of registration and deregistration will be performed.
- *
- * @see ServiceRegistry
- *
- */
-public interface RegisterableService {
-
-    /**
-     * Called when an object implementing this interface is added to
-     * the given <code>category</code> of the given
-     * <code>registry</code>.  The object may already be registered
-     * under another category or categories.
-     *
-     * @param registry a <code>ServiceRegistry</code> where this
-     * object has been registered.
-     * @param category a <code>Class</code> object indicating the
-     * registry category under which this object has been registered.
-     */
-    void onRegistration(ServiceRegistry registry, Class<?> category);
-
-    /**
-     * Called when an object implementing this interface is removed
-     * from the given <code>category</code> of the given
-     * <code>registry</code>.  The object may still be registered
-     * under another category or categories.
-     *
-     * @param registry a <code>ServiceRegistry</code> from which this
-     * object is being (wholly or partially) deregistered.
-     * @param category a <code>Class</code> object indicating the
-     * registry category from which this object is being deregistered.
-     */
-    void onDeregistration(ServiceRegistry registry, Class<?> category);
-}
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/ServiceRegistry.java b/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/ServiceRegistry.java
deleted file mode 100644
index 1794adb5de2..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/plugin/spi/ServiceRegistry.java
+++ /dev/null
@@ -1,861 +0,0 @@
-/*
-   Copyright (C) 2005-2012, by the President and Fellows of Harvard College.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-   Dataverse Network - A web application to share, preserve and analyze research data.
-   Developed at the Institute for Quantitative Social Science, Harvard University.
-   Version 3.0.
-*/
-package edu.harvard.iq.dataverse.ingest.plugin.spi;
-
-
-
-// This file was Taken out from openjdk-6-src-b16-24_apr_2009.tar.gz
-// http://download.java.net/openjdk/jdk6/promoted/b16/openjdk-6-src-b16-24_apr_2009.tar.gz
-// downloaded: 2009-05-07
-
-
-/*
- * Copyright 2000-2007 Sun Microsystems, Inc.  All Rights Reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Sun designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Sun in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
- * CA 95054 USA or visit www.sun.com if you need additional information or
- * have any questions.
- */
-
-//package javax.imageio.spi;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.ServiceLoader;
-
-/**
- * A registry for service provider instances.
- *
- * <p> A <i>service</i> is a well-known set of interfaces and (usually
- * abstract) classes.  A <i>service provider</i> is a specific
- * implementation of a service.  The classes in a provider typically
- * implement the interface or subclass the class defined by the
- * service itself.
- *
- * <p> Service providers are stored in one or more <i>categories</i>,
- * each of which is defined by a class of interface (described by a
- * <code>Class</code> object) that all of its members must implement.
- * The set of categories may be changed dynamically.
- *
- * <p> Only a single instance of a given leaf class (that is, the
- * actual class returned by <code>getClass()</code>, as opposed to any
- * inherited classes or interfaces) may be registered.  That is,
- * suppose that the
- * <code>com.mycompany.mypkg.GreenServiceProvider</code> class
- * implements the <code>com.mycompany.mypkg.MyService</code>
- * interface.  If a <code>GreenServiceProvider</code> instance is
- * registered, it will be stored in the category defined by the
- * <code>MyService</code> class.  If a new instance of
- * <code>GreenServiceProvider</code> is registered, it will replace
- * the previous instance.  In practice, service provider objects are
- * usually singletons so this behavior is appropriate.
- *
- * <p> To declare a service provider, a <code>services</code>
- * subdirectory is placed within the <code>META-INF</code> directory
- * that is present in every JAR file.  This directory contains a file
- * for each service provider interface that has one or more
- * implementation classes present in the JAR file.  For example, if
- * the JAR file contained a class named
- * <code>com.mycompany.mypkg.MyServiceImpl</code> which implements the
- * <code>javax.someapi.SomeService</code> interface, the JAR file
- * would contain a file named: <pre>
- * META-INF/services/javax.someapi.SomeService </pre>
- *
- * containing the line:
- *
- * <pre>
- * com.mycompany.mypkg.MyService
- * </pre>
- *
- * <p> The service provider classes should be to be lightweight and
- * quick to load.  Implementations of these interfaces should avoid
- * complex dependencies on other classes and on native code. The usual
- * pattern for more complex services is to register a lightweight
- * proxy for the heavyweight service.
- *
- * <p> An application may customize the contents of a registry as it
- * sees fit, so long as it has the appropriate runtime permission.
- *
- * <p> For more details on declaring service providers, and the JAR
- * format in general, see the <a
- * href="../../../../technotes/guides/jar/jar.html">
- * JAR File Specification</a>.
- *
- * @see RegisterableService
- *
- */
-public class ServiceRegistry {
-
-    // Class -> Registry
-    private Map<Class<?>, SubRegistry> categoryMap = new HashMap<>();
-
-    /**
-     * Constructs a <code>ServiceRegistry</code> instance with a
-     * set of categories taken from the <code>categories</code>
-     * argument.
-     *
-     * @param categories an <code>Iterator</code> containing
-     * <code>Class</code> objects to be used to define categories.
-     *
-     * @exception IllegalArgumentException if
-     * <code>categories</code> is <code>null</code>.
-     */
-    public ServiceRegistry(Iterator<Class<?>> categories) {
-        if (categories == null) {
-            throw new IllegalArgumentException("categories == null!");
-        }
-        while (categories.hasNext()) {
-            Class<?> category = categories.next();
-            SubRegistry reg = new SubRegistry(this, category);
-            categoryMap.put(category, reg);
-        }
-    }
-
-    // The following two methods expose functionality from
-    // sun.misc.Service.  If that class is made public, they may be
-    // removed.
-    //
-    // The sun.misc.ServiceConfigurationError class may also be
-    // exposed, in which case the references to 'an
-    // <code>Error</code>' below should be changed to 'a
-    // <code>ServiceConfigurationError</code>'.
-
-    /**
-     * Searches for implementations of a particular service class
-     * using the given class loader.
-     *
-     * <p> This method transforms the name of the given service class
-     * into a provider-configuration filename as described in the
-     * class comment and then uses the <code>getResources</code>
-     * method of the given class loader to find all available files
-     * with that name.  These files are then read and parsed to
-     * produce a list of provider-class names.  The iterator that is
-     * returned uses the given class loader to look up and then
-     * instantiate each element of the list.
-     *
-     * <p> Because it is possible for extensions to be installed into
-     * a running Java virtual machine, this method may return
-     * different results each time it is invoked.
-     *
-     * @param providerClass a <code>Class</code>object indicating the
-     * class or interface of the service providers being detected.
-     *
-     * @param loader the class loader to be used to load
-     * provider-configuration files and instantiate provider classes,
-     * or <code>null</code> if the system class loader (or, failing that
-     * the bootstrap class loader) is to be used.
-     *
-     * @return An <code>Iterator</code> that yields provider objects
-     * for the given service, in some arbitrary order.  The iterator
-     * will throw an <code>Error</code> if a provider-configuration
-     * file violates the specified format or if a provider class
-     * cannot be found and instantiated.
-     *
-     * @exception IllegalArgumentException if
-     * <code>providerClass</code> is <code>null</code>.
-     */
-    public static <T> Iterator<T> lookupProviders(Class<T> providerClass,
-                                                  ClassLoader loader)
-    {
-        if (providerClass == null) {
-            throw new IllegalArgumentException("providerClass == null!");
-        }
-        return ServiceLoader.load(providerClass, loader).iterator();
-    }
-
-    /**
-     * Locates and incrementally instantiates the available providers
-     * of a given service using the context class loader.  This
-     * convenience method is equivalent to:
-     *
-     * <pre>
-     *   ClassLoader cl = Thread.currentThread().getContextClassLoader();
-     *   return Service.providers(service, cl);
-     * </pre>
-     *
-     * @param providerClass a <code>Class</code>object indicating the
-     * class or interface of the service providers being detected.
-     *
-     * @return An <code>Iterator</code> that yields provider objects
-     * for the given service, in some arbitrary order.  The iterator
-     * will throw an <code>Error</code> if a provider-configuration
-     * file violates the specified format or if a provider class
-     * cannot be found and instantiated.
-     *
-     * @exception IllegalArgumentException if
-     * <code>providerClass</code> is <code>null</code>.
-     */
-    public static <T> Iterator<T> lookupProviders(Class<T> providerClass) {
-        if (providerClass == null) {
-            throw new IllegalArgumentException("providerClass == null!");
-        }
-        return ServiceLoader.load(providerClass).iterator();
-    }
-
-    /**
-     * Returns an <code>Iterator</code> of <code>Class</code> objects
-     * indicating the current set of categories.  The iterator will be
-     * empty if no categories exist.
-     *
-     * @return an <code>Iterator</code> containing
-     * <code>Class</code>objects.
-     */
-    public Iterator<Class<?>> getCategories() {
-        Set<Class<?>> keySet = categoryMap.keySet();
-        return keySet.iterator();
-    }
-
-    /**
-     * Returns an Iterator containing the subregistries to which the
-     * provider belongs.
-     */
-    private Iterator<SubRegistry> getSubRegistries(Object provider) {
-        List<SubRegistry> l = new ArrayList<>();
-        Iterator<Class<?>> iter = categoryMap.keySet().iterator();
-        while (iter.hasNext()) {
-            Class<?> c = iter.next();
-            if (c.isAssignableFrom(provider.getClass())) {
-                l.add(categoryMap.get(c));
-            }
-        }
-        return l.iterator();
-    }
-
-    /**
-     * Adds a service provider object to the registry.  The provider
-     * is associated with the given category.
-     *
-     * <p> If <code>provider</code> implements the
-     * <code>RegisterableService</code> interface, its
-     * <code>onRegistration</code> method will be called.  Its
-     * <code>onDeregistration</code> method will be called each time
-     * it is deregistered from a category, for example if a
-     * category is removed or the registry is garbage collected.
-     *
-     * @param provider the service provide object to be registered.
-     * @param category the category under which to register the
-     * provider.
-     *
-     * @return true if no provider of the same class was previously
-     * registered in the same category category.
-     *
-     * @exception IllegalArgumentException if <code>provider</code> is
-     * <code>null</code>.
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     * @exception ClassCastException if provider does not implement
-     * the <code>Class</code> defined by <code>category</code>.
-     */
-    public <T> boolean registerServiceProvider(T provider,
-                                               Class<T> category) {
-        if (provider == null) {
-            throw new IllegalArgumentException("provider == null!");
-        }
-        SubRegistry reg = categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        if (!category.isAssignableFrom(provider.getClass())) {
-            throw new ClassCastException();
-        }
-
-        return reg.registerServiceProvider(provider);
-    }
-
-    /**
-     * Adds a service provider object to the registry.  The provider
-     * is associated within each category present in the registry
-     * whose <code>Class</code> it implements.
-     *
-     * <p> If <code>provider</code> implements the
-     * <code>RegisterableService</code> interface, its
-     * <code>onRegistration</code> method will be called once for each
-     * category it is registered under.  Its
-     * <code>onDeregistration</code> method will be called each time
-     * it is deregistered from a category or when the registry is
-     * finalized.
-     *
-     * @param provider the service provider object to be registered.
-     *
-     * @exception IllegalArgumentException if
-     * <code>provider</code> is <code>null</code>.
-     */
-    public void registerServiceProvider(Object provider) {
-        if (provider == null) {
-            throw new IllegalArgumentException("provider == null!");
-        }
-        Iterator<SubRegistry> regs = getSubRegistries(provider);
-        while (regs.hasNext()) {
-            SubRegistry reg = regs.next();
-            reg.registerServiceProvider(provider);
-        }
-    }
-
-    /**
-     * Adds a set of service provider objects, taken from an
-     * <code>Iterator</code> to the registry.  Each provider is
-     * associated within each category present in the registry whose
-     * <code>Class</code> it implements.
-     *
-     * <p> For each entry of <code>providers</code> that implements
-     * the <code>RegisterableService</code> interface, its
-     * <code>onRegistration</code> method will be called once for each
-     * category it is registered under.  Its
-     * <code>onDeregistration</code> method will be called each time
-     * it is deregistered from a category or when the registry is
-     * finalized.
-     *
-     * @param providers an Iterator containing service provider
-     * objects to be registered.
-     *
-     * @exception IllegalArgumentException if <code>providers</code>
-     * is <code>null</code> or contains a <code>null</code> entry.
-     */
-    public void registerServiceProviders(Iterator<?> providers) {
-        if (providers == null) {
-            throw new IllegalArgumentException("provider == null!");
-        }
-        while (providers.hasNext()) {
-            registerServiceProvider(providers.next());
-        }
-    }
-
-    /**
-     * Removes a service provider object from the given category.  If
-     * the provider was not previously registered, nothing happens and
-     * <code>false</code> is returned.  Otherwise, <code>true</code>
-     * is returned.  If an object of the same class as
-     * <code>provider</code> but not equal (using <code>==</code>) to
-     * <code>provider</code> is registered, it will not be
-     * deregistered.
-     *
-     * <p> If <code>provider</code> implements the
-     * <code>RegisterableService</code> interface, its
-     * <code>onDeregistration</code> method will be called.
-     *
-     * @param provider the service provider object to be deregistered.
-     * @param category the category from which to deregister the
-     * provider.
-     *
-     * @return <code>true</code> if the provider was previously
-     * registered in the same category category,
-     * <code>false</code> otherwise.
-     *
-     * @exception IllegalArgumentException if <code>provider</code> is
-     * <code>null</code>.
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     * @exception ClassCastException if provider does not implement
-     * the class defined by <code>category</code>.
-     */
-    public <T> boolean deregisterServiceProvider(T provider,
-                                                 Class<T> category) {
-        if (provider == null) {
-            throw new IllegalArgumentException("provider == null!");
-        }
-        SubRegistry reg = categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        if (!category.isAssignableFrom(provider.getClass())) {
-            throw new ClassCastException();
-        }
-        return reg.deregisterServiceProvider(provider);
-    }
-
-    /**
-     * Removes a service provider object from all categories that
-     * contain it.
-     *
-     * @param provider the service provider object to be deregistered.
-     *
-     * @exception IllegalArgumentException if <code>provider</code> is
-     * <code>null</code>.
-     */
-    public void deregisterServiceProvider(Object provider) {
-        if (provider == null) {
-            throw new IllegalArgumentException("provider == null!");
-        }
-        Iterator<SubRegistry> regs = getSubRegistries(provider);
-        while (regs.hasNext()) {
-            SubRegistry reg = regs.next();
-            reg.deregisterServiceProvider(provider);
-        }
-    }
-
-    /**
-     * Returns <code>true</code> if <code>provider</code> is currently
-     * registered.
-     *
-     * @param provider the service provider object to be queried.
-     *
-     * @return <code>true</code> if the given provider has been
-     * registered.
-     *
-     * @exception IllegalArgumentException if <code>provider</code> is
-     * <code>null</code>.
-     */
-    public boolean contains(Object provider) {
-        if (provider == null) {
-            throw new IllegalArgumentException("provider == null!");
-        }
-        Iterator<SubRegistry> regs = getSubRegistries(provider);
-        while (regs.hasNext()) {
-            SubRegistry reg = regs.next();
-            if (reg.contains(provider)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    /**
-     * Returns an <code>Iterator</code> containing all registered
-     * service providers in the given category.  If
-     * <code>useOrdering</code> is <code>false</code>, the iterator
-     * will return all of the server provider objects in an arbitrary
-     * order.  Otherwise, the ordering will respect any pairwise
-     * orderings that have been set.  If the graph of pairwise
-     * orderings contains cycles, any providers that belong to a cycle
-     * will not be returned.
-     *
-     * @param category the category to be retrieved from.
-     * @param useOrdering <code>true</code> if pairwise orderings
-     * should be taken account in ordering the returned objects.
-     *
-     * @return an <code>Iterator</code> containing service provider
-     * objects from the given category, possibly in order.
-     *
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     */
-    public <T> Iterator<T> getServiceProviders(Class<T> category,
-                                               boolean useOrdering) {
-        SubRegistry reg = categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        return reg.getServiceProviders(useOrdering);
-    }
-
-    /**
-     * A simple filter interface used by
-     * <code>ServiceRegistry.getServiceProviders</code> to select
-     * providers matching an arbitrary criterion.  Classes that
-     * implement this interface should be defined in order to make use
-     * of the <code>getServiceProviders</code> method of
-     * <code>ServiceRegistry</code> that takes a <code>Filter</code>.
-     *
-     * @see ServiceRegistry#getServiceProviders(Class, ServiceRegistry.Filter, boolean)
-     */
-    public interface Filter {
-
-        /**
-         * Returns <code>true</code> if the given
-         * <code>provider</code> object matches the criterion defined
-         * by this <code>Filter</code>.
-         *
-         * @param provider a service provider <code>Object</code>.
-         *
-         * @return true if the provider matches the criterion.
-         */
-        boolean filter(Object provider);
-    }
-
-    /**
-     * Returns an <code>Iterator</code> containing service provider
-     * objects within a given category that satisfy a criterion
-     * imposed by the supplied <code>ServiceRegistry.Filter</code>
-     * object's <code>filter</code> method.
-     *
-     * <p> The <code>useOrdering</code> argument controls the
-     * ordering of the results using the same rules as
-     * <code>getServiceProviders(Class, boolean)</code>.
-     *
-     * @param category the category to be retrieved from.
-     * @param filter an instance of <code>ServiceRegistry.Filter</code>
-     * whose <code>filter</code> method will be invoked.
-     * @param useOrdering <code>true</code> if pairwise orderings
-     * should be taken account in ordering the returned objects.
-     *
-     * @return an <code>Iterator</code> containing service provider
-     * objects from the given category, possibly in order.
-     *
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     */
-    public <T> Iterator<T> getServiceProviders(Class<T> category,
-                                               Filter filter,
-                                               boolean useOrdering) {
-        SubRegistry reg = categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        Iterator iter = getServiceProviders(category, useOrdering);
-        return new FilterIterator(iter, filter);
-    }
-
-    /**
-     * Returns the currently registered service provider object that
-     * is of the given class type.  At most one object of a given
-     * class is allowed to be registered at any given time.  If no
-     * registered object has the desired class type, <code>null</code>
-     * is returned.
-     *
-     * @param providerClass the <code>Class</code> of the desired
-     * service provider object.
-     *
-     * @return a currently registered service provider object with the
-     * desired <code>Class</code>type, or <code>null</code> is none is
-     * present.
-     *
-     * @exception IllegalArgumentException if <code>providerClass</code> is
-     * <code>null</code>.
-     */
-    public <T> T getServiceProviderByClass(Class<T> providerClass) {
-        if (providerClass == null) {
-            throw new IllegalArgumentException("providerClass == null!");
-        }
-        for (Class c : categoryMap.keySet()) {
-            if (c.isAssignableFrom(providerClass)) {
-                SubRegistry reg = (SubRegistry)categoryMap.get(c);
-                T provider = reg.getServiceProviderByClass(providerClass);
-                if (provider != null) {
-                    return provider;
-                }
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Sets a pairwise ordering between two service provider objects
-     * within a given category.  If one or both objects are not
-     * currently registered within the given category, or if the
-     * desired ordering is already set, nothing happens and
-     * <code>false</code> is returned.  If the providers previously
-     * were ordered in the reverse direction, that ordering is
-     * removed.
-     *
-     * <p> The ordering will be used by the
-     * <code>getServiceProviders</code> methods when their
-     * <code>useOrdering</code> argument is <code>true</code>.
-     *
-     * @param category a <code>Class</code> object indicating the
-     * category under which the preference is to be established.
-     * @param firstProvider the preferred provider.
-     * @param secondProvider the provider to which
-     * <code>firstProvider</code> is preferred.
-     *
-     * @return <code>true</code> if a previously unset ordering
-     * was established.
-     *
-     * @exception IllegalArgumentException if either provider is
-     * <code>null</code> or they are the same object.
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     */
-    public <T> boolean setOrdering(Class<T> category,
-                                   T firstProvider,
-                                   T secondProvider) {
-        if (firstProvider == null || secondProvider == null) {
-            throw new IllegalArgumentException("provider is null!");
-        }
-        if (firstProvider == secondProvider) {
-            throw new IllegalArgumentException("providers are the same!");
-        }
-        SubRegistry reg = (SubRegistry)categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        if (reg.contains(firstProvider) &&
-            reg.contains(secondProvider)) {
-            return reg.setOrdering(firstProvider, secondProvider);
-        }
-        return false;
-    }
-
-    /**
-     * Sets a pairwise ordering between two service provider objects
-     * within a given category.  If one or both objects are not
-     * currently registered within the given category, or if no
-     * ordering is currently set between them, nothing happens
-     * and <code>false</code> is returned.
-     *
-     * <p> The ordering will be used by the
-     * <code>getServiceProviders</code> methods when their
-     * <code>useOrdering</code> argument is <code>true</code>.
-     *
-     * @param category a <code>Class</code> object indicating the
-     * category under which the preference is to be disestablished.
-     * @param firstProvider the formerly preferred provider.
-     * @param secondProvider the provider to which
-     * <code>firstProvider</code> was formerly preferred.
-     *
-     * @return <code>true</code> if a previously set ordering was
-     * disestablished.
-     *
-     * @exception IllegalArgumentException if either provider is
-     * <code>null</code> or they are the same object.
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     */
-    public <T> boolean unsetOrdering(Class<T> category,
-                                     T firstProvider,
-                                     T secondProvider) {
-        if (firstProvider == null || secondProvider == null) {
-            throw new IllegalArgumentException("provider is null!");
-        }
-        if (firstProvider == secondProvider) {
-            throw new IllegalArgumentException("providers are the same!");
-        }
-        SubRegistry reg = (SubRegistry)categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        if (reg.contains(firstProvider) &&
-            reg.contains(secondProvider)) {
-            return reg.unsetOrdering(firstProvider, secondProvider);
-        }
-        return false;
-    }
-
-    /**
-     * Deregisters all service provider object currently registered
-     * under the given category.
-     *
-     * @param category the category to be emptied.
-     *
-     * @exception IllegalArgumentException if there is no category
-     * corresponding to <code>category</code>.
-     */
-    public void deregisterAll(Class<?> category) {
-        SubRegistry reg = (SubRegistry)categoryMap.get(category);
-        if (reg == null) {
-            throw new IllegalArgumentException("category unknown!");
-        }
-        reg.clear();
-    }
-
-    /**
-     * Deregisters all currently registered service providers from all
-     * categories.
-     */
-    public void deregisterAll() {
-        Iterator iter = categoryMap.values().iterator();
-        while (iter.hasNext()) {
-            SubRegistry reg = (SubRegistry)iter.next();
-            reg.clear();
-        }
-    }
-
-    /**
-     * Finalizes this object prior to garbage collection.  The
-     * <code>deregisterAll</code> method is called to deregister all
-     * currently registered service providers.  This method should not
-     * be called from application code.
-     *
-     * @exception Throwable if an error occurs during superclass
-     * finalization.
-     */
-    public void finalize() throws Throwable {
-        deregisterAll();
-        super.finalize();
-    }
-}
-
-
-/**
- * A portion of a registry dealing with a single superclass or
- * interface.
- */
-class SubRegistry {
-
-    ServiceRegistry registry;
-
-    Class<?> category;
-
-    // Provider Objects organized by partial oridering
-    PartiallyOrderedSet poset = new PartiallyOrderedSet();
-
-    // Class -> Provider Object of that class
-    Map<Class<?>,Object> map = new HashMap<>();
-
-    public SubRegistry(ServiceRegistry registry, Class<?> category) {
-        this.registry = registry;
-        this.category = category;
-    }
-
-    public boolean registerServiceProvider(Object provider) {
-        Object oprovider = map.get(provider.getClass());
-        boolean present =  oprovider != null;
-
-        if (present) {
-            deregisterServiceProvider(oprovider);
-        }
-        map.put(provider.getClass(), provider);
-        poset.add(provider);
-        if (provider instanceof RegisterableService) {
-            RegisterableService rs = (RegisterableService)provider;
-            rs.onRegistration(registry, category);
-        }
-
-        return !present;
-    }
-
-    /**
-     * If the provider was not previously registered, do nothing.
-     *
-     * @return true if the provider was previously registered.
-     */
-    public boolean deregisterServiceProvider(Object provider) {
-        Object oprovider = map.get(provider.getClass());
-
-        if (provider == oprovider) {
-            map.remove(provider.getClass());
-            poset.remove(provider);
-            if (provider instanceof RegisterableService) {
-                RegisterableService rs = (RegisterableService)provider;
-                rs.onDeregistration(registry, category);
-            }
-
-            return true;
-        }
-        return false;
-    }
-
-    public boolean contains(Object provider) {
-        Object oprovider = map.get(provider.getClass());
-        return oprovider == provider;
-    }
-
-    public boolean setOrdering(Object firstProvider,
-                               Object secondProvider) {
-        return poset.setOrdering(firstProvider, secondProvider);
-    }
-
-    public boolean unsetOrdering(Object firstProvider,
-                                 Object secondProvider) {
-        return poset.unsetOrdering(firstProvider, secondProvider);
-    }
-
-    public Iterator getServiceProviders(boolean useOrdering) {
-        if (useOrdering) {
-            return poset.iterator();
-        } else {
-            return map.values().iterator();
-        }
-    }
-
-    public <T> T getServiceProviderByClass(Class<T> providerClass) {
-        return (T)map.get(providerClass);
-    }
-
-    public void clear() {
-        Iterator iter = map.values().iterator();
-        while (iter.hasNext()) {
-            Object provider = iter.next();
-            iter.remove();
-
-            if (provider instanceof RegisterableService) {
-                RegisterableService rs = (RegisterableService)provider;
-                rs.onDeregistration(registry, category);
-            }
-        }
-        poset.clear();
-    }
-
-    public void finalize() {
-        clear();
-    }
-}
-
-
-/**
- * A class for wrapping <code>Iterators</code> with a filter function.
- * This provides an iterator for a subset without duplication.
- */
-class FilterIterator<T> implements Iterator<T> {
-
-    private Iterator<T> iter;
-    private ServiceRegistry.Filter filter;
-
-    private T next = null;
-
-    public FilterIterator(Iterator<T> iter,
-                          ServiceRegistry.Filter filter) {
-        this.iter = iter;
-        this.filter = filter;
-        advance();
-    }
-
-    private void advance() {
-        while (iter.hasNext()) {
-            T elt = iter.next();
-            if (filter.filter(elt)) {
-                next = elt;
-                return;
-            }
-        }
-
-        next = null;
-    }
-
-    public boolean hasNext() {
-        return next != null;
-    }
-
-    public T next() {
-        if (next == null) {
-            throw new NoSuchElementException();
-        }
-        T o = next;
-        advance();
-        return o;
-    }
-
-    public void remove() {
-        throw new UnsupportedOperationException();
-    }
-}

From 6cfe538cac9ed16873e277d117eacfd1e4fd7fd9 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Tue, 18 Oct 2022 00:03:48 +0200
Subject: [PATCH 004/922] Update HTTP URL to HTTPS URL

---
 .../source/_static/navbarscroll.js            |  2 +-
 .../source/_templates/navbar.html             | 32 +++++++++----------
 doc/sphinx-guides/source/admin/monitoring.rst |  2 +-
 .../source/api/client-libraries.rst           |  6 ++--
 .../source/api/external-tools.rst             |  4 +--
 .../source/api/getting-started.rst            |  2 +-
 doc/sphinx-guides/source/api/intro.rst        |  2 +-
 doc/sphinx-guides/source/api/native-api.rst   | 12 +++----
 doc/sphinx-guides/source/api/sword.rst        | 12 +++----
 doc/sphinx-guides/source/conf.py              |  2 +-
 .../source/developers/dev-environment.rst     |  8 ++---
 .../source/developers/documentation.rst       |  6 ++--
 doc/sphinx-guides/source/developers/intro.rst |  6 ++--
 .../source/developers/testing.rst             |  8 ++---
 doc/sphinx-guides/source/developers/tools.rst |  8 ++---
 .../source/developers/unf/index.rst           |  2 +-
 .../source/developers/unf/unf-v3.rst          |  6 ++--
 .../source/developers/unf/unf-v6.rst          |  2 +-
 .../source/developers/version-control.rst     |  2 +-
 doc/sphinx-guides/source/index.rst            |  4 +--
 .../source/installation/config.rst            | 26 +++++++--------
 .../source/installation/installation-main.rst |  6 ++--
 .../source/installation/intro.rst             |  4 +--
 .../source/installation/oauth2.rst            |  2 +-
 .../source/installation/oidc.rst              |  2 +-
 .../source/installation/prerequisites.rst     |  4 +--
 .../source/installation/shibboleth.rst        |  6 ++--
 .../source/style/foundations.rst              | 16 +++++-----
 doc/sphinx-guides/source/style/patterns.rst   | 28 ++++++++--------
 doc/sphinx-guides/source/user/account.rst     |  2 +-
 doc/sphinx-guides/source/user/appendix.rst    | 10 +++---
 .../source/user/dataset-management.rst        |  6 ++--
 .../user/tabulardataingest/ingestprocess.rst  |  4 +--
 33 files changed, 122 insertions(+), 122 deletions(-)

diff --git a/doc/sphinx-guides/source/_static/navbarscroll.js b/doc/sphinx-guides/source/_static/navbarscroll.js
index 66c9d4d7995..735f80870cd 100644
--- a/doc/sphinx-guides/source/_static/navbarscroll.js
+++ b/doc/sphinx-guides/source/_static/navbarscroll.js
@@ -1,6 +1,6 @@
 /*  
     Use to fix hidden section headers behind the navbar when using links with targets
-    See: http://stackoverflow.com/questions/10732690/offsetting-an-html-anchor-to-adjust-for-fixed-header    
+    See: https://stackoverflow.com/questions/10732690/offsetting-an-html-anchor-to-adjust-for-fixed-header    
 */
 $jqTheme(document).ready(function() {
   $jqTheme('a[href*="#"]:not([href="#"])').on('click', function() {
diff --git a/doc/sphinx-guides/source/_templates/navbar.html b/doc/sphinx-guides/source/_templates/navbar.html
index 538cccf74d7..c7b81dcb937 100644
--- a/doc/sphinx-guides/source/_templates/navbar.html
+++ b/doc/sphinx-guides/source/_templates/navbar.html
@@ -15,7 +15,7 @@
                 <span class="icon-bar"></span>
                 <span class="icon-bar"></span>
             </button>
-            <a class="navbar-brand" href="http://dataverse.org" id="dataverse-org-homepage-url">Dataverse Project</a>
+            <a class="navbar-brand" href="https://dataverse.org" id="dataverse-org-homepage-url">Dataverse Project</a>
         </div>
         <!-- Collect the nav links, forms, and other content for toggling -->
 
@@ -24,15 +24,15 @@
                 <li class="dropdown">
                     <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">About <span class="caret"></span></a>
                     <ul class="dropdown-menu">
-                        <li><a target="_blank" href="http://dataverse.org/about">About the Project</a></li>
-                        <li><a target="_blank" href="http://dataverse.org/add-data">Add Data</a></li>
-                        <li><a target="_blank" href="http://dataverse.org/blog">Blog</a></li>
-                        <li><a target="_blank" href="http://dataverse.org/presentations">Presentations</a></li>
-                        <li><a target="_blank" href="http://dataverse.org/publications">Publications</a></li>
+                        <li><a target="_blank" href="https://dataverse.org/about">About the Project</a></li>
+                        <li><a target="_blank" href="https://dataverse.org/add-data">Add Data</a></li>
+                        <li><a target="_blank" href="https://dataverse.org/blog">Blog</a></li>
+                        <li><a target="_blank" href="https://dataverse.org/presentations">Presentations</a></li>
+                        <li><a target="_blank" href="https://dataverse.org/publications">Publications</a></li>
                     </ul>
                 </li>
                 <li>
-                    <a target="_blank" href="http://dataverse.org/">
+                    <a target="_blank" href="https://dataverse.org/">
                         Community
                     </a>
                 </li>
@@ -49,18 +49,18 @@
                 <li>
                     <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Software <span class="caret"></span></a>
                     <ul class="dropdown-menu">
-                        <li><a target="_blank" href="http://dataverse.org/software-features">Features</a></li>
-                        <li><a target="_blank" href="http://github.com/IQSS/dataverse">Source Code</a></li>
-                        <li><a target="_blank" href="http://guides.dataverse.org/en/latest/user/index.html">User Guide</a></li>
-                        <li><a target="_blank" href="http://guides.dataverse.org/en/latest/installation/index.html">Installation Guide</a></li>
-                        <li><a target="_blank" href="http://guides.dataverse.org/en/latest/api/index.html">API Guide</a></li>
-                        <li><a target="_blank" href="http://guides.dataverse.org/en/latest/developers/index.html">Developer Guide</a></li>
-                        <li><a target="_blank" href="http://guides.dataverse.org/en/latest/style/index.html">Style Guide</a></li>
-                        <li><a target="_blank" href="http://guides.dataverse.org/en/latest/admin/index.html">Admin Guide</a></li>
+                        <li><a target="_blank" href="https://dataverse.org/software-features">Features</a></li>
+                        <li><a target="_blank" href="https://github.com/IQSS/dataverse">Source Code</a></li>
+                        <li><a target="_blank" href="https://guides.dataverse.org/en/latest/user/index.html">User Guide</a></li>
+                        <li><a target="_blank" href="https://guides.dataverse.org/en/latest/installation/index.html">Installation Guide</a></li>
+                        <li><a target="_blank" href="https://guides.dataverse.org/en/latest/api/index.html">API Guide</a></li>
+                        <li><a target="_blank" href="https://guides.dataverse.org/en/latest/developers/index.html">Developer Guide</a></li>
+                        <li><a target="_blank" href="https://guides.dataverse.org/en/latest/style/index.html">Style Guide</a></li>
+                        <li><a target="_blank" href="https://guides.dataverse.org/en/latest/admin/index.html">Admin Guide</a></li>
                     </ul>
                 </li>
                 <li>
-                    <a target="_blank" href="http://dataverse.org/">
+                    <a target="_blank" href="https://dataverse.org/">
                         Contact
                     </a>
                 </li>
diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst
index a4affda1302..e902d5fdcc9 100644
--- a/doc/sphinx-guides/source/admin/monitoring.rst
+++ b/doc/sphinx-guides/source/admin/monitoring.rst
@@ -14,7 +14,7 @@ In production you'll want to monitor the usual suspects such as CPU, memory, fre
 Munin
 +++++
 
-http://munin-monitoring.org says, "A default installation provides a lot of graphs with almost no work." From RHEL or CentOS 7, you can try the following steps.
+https://munin-monitoring.org says, "A default installation provides a lot of graphs with almost no work." From RHEL or CentOS 7, you can try the following steps.
 
 Enable the EPEL yum repo (if you haven't already):
 
diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index 634f03a8125..388a9d641ed 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -13,7 +13,7 @@ Python
 
 There are two Python modules for interacting with Dataverse Software APIs.
 
-`pyDataverse <https://github.com/gdcc/pyDataverse>`_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger <http://stefankasberger.at>`_ from `AUSSDA - The Austrian Social Science Data Archive <https://aussda.at>`_.  
+`pyDataverse <https://github.com/gdcc/pyDataverse>`_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger <https://stefankasberger.at>`_ from `AUSSDA - The Austrian Social Science Data Archive <https://aussda.at>`_.  
 
 `dataverse-client-python <https://github.com/IQSS/dataverse-client-python>`_ had its initial release in 2015. `Robert Liebowitz <https://github.com/rliebz>`_ created this library while at the `Center for Open Science (COS) <https://centerforopenscience.org>`_ and the COS uses it to integrate the `Open Science Framework (OSF) <https://osf.io>`_ with a Dataverse installation via an add-on which itself is open source and listed on the :doc:`/api/apps` page.
 
@@ -30,14 +30,14 @@ R
 https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse Software APIs. The latest release can be installed from `CRAN <https://cran.r-project.org/package=dataverse>`_. 
 The R client can search and download datasets. It is useful when automatically (instead of manually) downloading data files as part of a script. For bulk edit and upload operations, we currently recommend pyDataverse.
 
-The package is currently maintained by  `Shiro Kuriwaki <https://github.com/kuriwaki>`_. It was originally created by `Thomas Leeper <http://thomasleeper.com>`_ and then formerly maintained by `Will Beasley <https://github.com/wibeasley>`_.
+The package is currently maintained by  `Shiro Kuriwaki <https://github.com/kuriwaki>`_. It was originally created by `Thomas Leeper <https://thomasleeper.com>`_ and then formerly maintained by `Will Beasley <https://github.com/wibeasley>`_.
 
 Java
 ----
 
 https://github.com/IQSS/dataverse-client-java is the official Java library for Dataverse Software APIs.
 
-`Richard Adams <http://www.researchspace.com/electronic-lab-notebook/about_us_team.html>`_ from `ResearchSpace <http://www.researchspace.com>`_ created and maintains this library.
+`Richard Adams <https://www.researchspace.com/electronic-lab-notebook/about_us_team.html>`_ from `ResearchSpace <https://www.researchspace.com>`_ created and maintains this library.
 
 Ruby
 ----
diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst
index d72a6f62004..8c6c9fa8d46 100644
--- a/doc/sphinx-guides/source/api/external-tools.rst
+++ b/doc/sphinx-guides/source/api/external-tools.rst
@@ -11,7 +11,7 @@ Introduction
 
 External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse Software.
 
-Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org
+Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=https://demo.dataverse.org
 
 In short, you will be creating a manifest in JSON format that describes not only how to construct URLs for your tool, but also what types of files your tool operates on, where it should appear in the Dataverse installation web interfaces, etc. 
 
@@ -94,7 +94,7 @@ Terminology
 
     toolParameters               **Query parameters** are supported and described below.
 
-    queryParameters              **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org``.
+    queryParameters              **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=https://demo.dataverse.org``.
 
     query parameter keys         An **arbitrary string** to associate with a value that is populated with a reserved word (described below). As the author of the tool, you have control over what "key" you would like to be passed to your tool. For example, if you want to have your tool receive and operate on the query parameter "dataverseFileId=42" instead of just "fileId=42", that's fine.
 
diff --git a/doc/sphinx-guides/source/api/getting-started.rst b/doc/sphinx-guides/source/api/getting-started.rst
index c465b726421..fd7c561cdf0 100644
--- a/doc/sphinx-guides/source/api/getting-started.rst
+++ b/doc/sphinx-guides/source/api/getting-started.rst
@@ -9,7 +9,7 @@ If you are a researcher or curator who wants to automate parts of your workflow,
 Servers You Can Test With
 -------------------------
 
-Rather than using a production Dataverse installation, API users are welcome to use http://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server.  
+Rather than using a production Dataverse installation, API users are welcome to use https://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server.  
 
 If you would rather have full control over your own test server, deployments to AWS, Docker, Vagrant, and more are covered in the :doc:`/developers/index` and the :doc:`/installation/index`.
 
diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst
index 933932cd7b9..6c61bb8c20d 100755
--- a/doc/sphinx-guides/source/api/intro.rst
+++ b/doc/sphinx-guides/source/api/intro.rst
@@ -237,7 +237,7 @@ Dataverse Software API questions are on topic in all the usual places:
 
 - The dataverse-community Google Group: https://groups.google.com/forum/#!forum/dataverse-community
 - The Dataverse Project community calls: https://dataverse.org/community-calls
-- The Dataverse Project chat room: http://chat.dataverse.org 
+- The Dataverse Project chat room: https://chat.dataverse.org 
 - The Dataverse Project ticketing system: support@dataverse.org
 
 After your question has been answered, you are welcome to help improve the :doc:`faq` section of this guide.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 93e1c36f179..578b35011ff 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -9,7 +9,7 @@ The Dataverse Software exposes most of its GUI functionality via a REST-based AP
 
 .. _CORS: https://www.w3.org/TR/cors/
 
-.. warning:: The Dataverse Software's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded.
+.. warning:: The Dataverse Software's API is versioned at the URI - all API calls may include the version number like so: ``https://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded.
 
 .. contents:: |toctitle|
     :local:
@@ -508,7 +508,7 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X PUT https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot
 
-.. note:: Previous endpoints ``$SERVER/api/dataverses/$id/metadatablocks/:isRoot`` and ``POST http://$SERVER/api/dataverses/$id/metadatablocks/:isRoot?key=$apiKey`` are deprecated, but supported.
+.. note:: Previous endpoints ``$SERVER/api/dataverses/$id/metadatablocks/:isRoot`` and ``POST https://$SERVER/api/dataverses/$id/metadatablocks/:isRoot?key=$apiKey`` are deprecated, but supported.
 
 
 .. _create-dataset-command: 
@@ -720,7 +720,7 @@ Getting its draft version:
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
 
-  curl -H "X-Dataverse-key:$API_TOKEN" http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER
+  curl -H "X-Dataverse-key:$API_TOKEN" https://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER
 
 The fully expanded example above (without environment variables) looks like this:
 
@@ -2226,7 +2226,7 @@ The fully expanded example above (without environment variables) looks like this
 Currently the following methods are used to detect file types:
 
 - The file type detected by the browser (or sent via API).
-- JHOVE: http://jhove.openpreservation.org
+- JHOVE: https://jhove.openpreservation.org
 - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``.
 - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``.
 
@@ -2413,7 +2413,7 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
     -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
-    http://demo.dataverse.org/api/files/24/metadata
+    https://demo.dataverse.org/api/files/24/metadata
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -2614,7 +2614,7 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/prov-freeform?persistentId=doi:10.5072/FK2/AAA000" -H "Content-type:application/json" --upload-file provenance.json
 
-See a sample JSON file :download:`file-provenance.json <../_static/api/file-provenance.json>` from http://openprovenance.org (c.f. Huynh, Trung Dong and Moreau, Luc (2014) ProvStore: a public provenance repository. At 5th International Provenance and Annotation Workshop (IPAW'14), Cologne, Germany, 09-13 Jun 2014. pp. 275-277).
+See a sample JSON file :download:`file-provenance.json <../_static/api/file-provenance.json>` from https://openprovenance.org (c.f. Huynh, Trung Dong and Moreau, Luc (2014) ProvStore: a public provenance repository. At 5th International Provenance and Annotation Workshop (IPAW'14), Cologne, Germany, 09-13 Jun 2014. pp. 275-277).
 
 Delete Provenance JSON for an uploaded file
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst
index 11b43e98774..c9ac83bc204 100755
--- a/doc/sphinx-guides/source/api/sword.rst
+++ b/doc/sphinx-guides/source/api/sword.rst
@@ -9,19 +9,19 @@ SWORD_ stands for "Simple Web-service Offering Repository Deposit" and is a "pro
 About
 -----
 
-Introduced in Dataverse Network (DVN) `3.6 <http://guides.dataverse.org/en/3.6.2/dataverse-api-main.html#data-deposit-api>`_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`.
+Introduced in Dataverse Network (DVN) `3.6 <https://guides.dataverse.org/en/3.6.2/dataverse-api-main.html#data-deposit-api>`_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`.
 
-The Dataverse Software implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc.
+The Dataverse Software implements most of SWORDv2_, which is specified at https://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc.
 
 As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse Software 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in a Dataverse installation.
 
-.. _SWORD: http://en.wikipedia.org/wiki/SWORD_%28protocol%29
+.. _SWORD: https://en.wikipedia.org/wiki/SWORD_%28protocol%29
 
 .. _SWORDv2: http://swordapp.org/sword-v2/sword-v2-specifications/
 
 .. _RFC 5023: https://tools.ietf.org/html/rfc5023
 
-.. _SWORDv2 specification: http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html
+.. _SWORDv2 specification: https://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html
 
 .. _sword-auth:
 
@@ -86,7 +86,7 @@ New features as of v1.1
 
 - "Contact E-mail" is automatically populated from dataset owner's email.
 
-- "Subject" uses our controlled vocabulary list of subjects. This list is in the Citation Metadata of our User Guide > `Metadata References <http://guides.dataverse.org/en/latest/user/appendix.html#metadata-references>`_. Otherwise, if a term does not match our controlled vocabulary list, it will put any subject terms in "Keyword". If Subject is empty it is automatically populated with "N/A".
+- "Subject" uses our controlled vocabulary list of subjects. This list is in the Citation Metadata of our User Guide > `Metadata References <https://guides.dataverse.org/en/latest/user/appendix.html#metadata-references>`_. Otherwise, if a term does not match our controlled vocabulary list, it will put any subject terms in "Keyword". If Subject is empty it is automatically populated with "N/A".
 
 - Zero-length files are now allowed (but not necessarily encouraged).
 
@@ -127,7 +127,7 @@ Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse Project DB Element Cr
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
 |dcterms:creator              |         authorName (LastName, FirstName)     |       Y      |  Author(s) for the Dataset.                                                                                                                                 |
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
-|dcterms:subject              |   subject (Controlled Vocabulary) OR keyword |       Y      |  Controlled Vocabulary list is in our User Guide > `Metadata References <http://guides.dataverse.org/en/latest/user/appendix.html#metadata-references>`_.   |                                                                                                                
+|dcterms:subject              |   subject (Controlled Vocabulary) OR keyword |       Y      |  Controlled Vocabulary list is in our User Guide > `Metadata References <https://guides.dataverse.org/en/latest/user/appendix.html#metadata-references>`_.   |                                                                                                                
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
 |dcterms:description          |              dsDescriptionValue              |       Y      |  Describing the purpose, scope or nature of the Dataset. Can also use dcterms:abstract.                                                                     |
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py
index 880ed561720..5ff538c3c46 100755
--- a/doc/sphinx-guides/source/conf.py
+++ b/doc/sphinx-guides/source/conf.py
@@ -432,7 +432,7 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {'https://docs.python.org/': None}
 # Suppress "WARNING: unknown mimetype for ..." https://github.com/IQSS/dataverse/issues/3391
 suppress_warnings = ['epub.unknown_project_files']
 rst_prolog = """
diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst
index e44a70a405f..2139b85c64a 100755
--- a/doc/sphinx-guides/source/developers/dev-environment.rst
+++ b/doc/sphinx-guides/source/developers/dev-environment.rst
@@ -34,7 +34,7 @@ On Linux, you are welcome to use the OpenJDK available from package managers.
 Install Netbeans or Maven
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-NetBeans IDE is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven.
+NetBeans IDE is recommended, and can be downloaded from https://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven.
 
 Below we describe how to build the Dataverse Software war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section.
 
@@ -74,7 +74,7 @@ On Mac, run this command:
 
 ``brew install jq``
 
-On Linux, install ``jq`` from your package manager or download a binary from http://stedolan.github.io/jq/
+On Linux, install ``jq`` from your package manager or download a binary from https://stedolan.github.io/jq/
 
 Install Payara
 ~~~~~~~~~~~~~~
@@ -117,7 +117,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager
 Install Solr
 ~~~~~~~~~~~~
 
-`Solr <http://lucene.apache.org/solr/>`_ 8.11.1 is required.
+`Solr <https://lucene.apache.org/solr/>`_ 8.11.1 is required.
 
 To install Solr, execute the following commands:
 
@@ -127,7 +127,7 @@ To install Solr, execute the following commands:
 
 ``cd /usr/local/solr``
 
-``curl -O http://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
+``curl -O https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
 
 ``tar xvfz solr-8.11.1.tgz``
 
diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst
index b20fd112533..46fc268461b 100755
--- a/doc/sphinx-guides/source/developers/documentation.rst
+++ b/doc/sphinx-guides/source/developers/documentation.rst
@@ -34,7 +34,7 @@ If you would like to read more about the Dataverse Project's use of GitHub, plea
 Building the Guides with Sphinx
 -------------------------------
 
-The Dataverse guides are written using Sphinx (http://sphinx-doc.org). We recommend installing Sphinx and building the guides locally so you can get an accurate preview of your changes.
+The Dataverse guides are written using Sphinx (https://sphinx-doc.org). We recommend installing Sphinx and building the guides locally so you can get an accurate preview of your changes.
 
 Installing Sphinx
 ~~~~~~~~~~~~~~~~~
@@ -58,7 +58,7 @@ In some parts of the documentation, graphs are rendered as images using the Sphi
 
 Building the guides requires the ``dot`` executable from GraphViz.
 
-This requires having `GraphViz <http://graphviz.org>`_ installed and either having ``dot`` on the path or
+This requires having `GraphViz <https://graphviz.org>`_ installed and either having ``dot`` on the path or
 `adding options to the make call <https://groups.google.com/forum/#!topic/sphinx-users/yXgNey_0M3I>`_.
 
 Editing and Building the Guides
@@ -67,7 +67,7 @@ Editing and Building the Guides
 To edit the existing documentation:
 
 - Create a branch (see :ref:`how-to-make-a-pull-request`).
-- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to http://guides.dataverse.org.
+- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to https://guides.dataverse.org.
 - Using your preferred text editor, open and edit the necessary files, or create new ones.
 
 Once you are done, open a terminal, change directories to ``doc/sphinx-guides``, activate (or reactivate) your Python virtual environment, and build the guides.
diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst
index 7f4e8c1ba34..6469a43b5ab 100755
--- a/doc/sphinx-guides/source/developers/intro.rst
+++ b/doc/sphinx-guides/source/developers/intro.rst
@@ -2,7 +2,7 @@
 Introduction
 ============
 
-Welcome! `The Dataverse Project <http://dataverse.org>`_ is an `open source <https://github.com/IQSS/dataverse/blob/master/LICENSE.md>`_ project that loves `contributors <https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md>`_!
+Welcome! `The Dataverse Project <https://dataverse.org>`_ is an `open source <https://github.com/IQSS/dataverse/blob/master/LICENSE.md>`_ project that loves `contributors <https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md>`_!
 
 .. contents:: |toctitle|
 	:local:
@@ -19,7 +19,7 @@ To get started, you'll want to set up your :doc:`dev-environment` and make sure
 Getting Help
 ------------
 
-If you have any questions at all, please reach out to other developers via the channels listed in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md such as http://chat.dataverse.org, the `dataverse-dev <https://groups.google.com/forum/#!forum/dataverse-dev>`_ mailing list, `community calls <https://dataverse.org/community-calls>`_, or support@dataverse.org.
+If you have any questions at all, please reach out to other developers via the channels listed in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md such as https://chat.dataverse.org, the `dataverse-dev <https://groups.google.com/forum/#!forum/dataverse-dev>`_ mailing list, `community calls <https://dataverse.org/community-calls>`_, or support@dataverse.org.
 
 .. _core-technologies:
 
@@ -52,7 +52,7 @@ Related Guides
 
 If you are a developer who wants to make use of the Dataverse Software APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`.
 
-If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`. We validate the installation scripts with :doc:`/developers/tools` such as `Vagrant <http://vagrantup.com>`_ and Docker (see the :doc:`containers` section).
+If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`. We validate the installation scripts with :doc:`/developers/tools` such as `Vagrant <https://vagrantup.com>`_ and Docker (see the :doc:`containers` section).
 
 Related Projects
 ----------------
diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 4b3d5fd0a55..132120291c2 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -46,7 +46,7 @@ The main takeaway should be that we care about unit testing enough to measure th
 Writing Unit Tests with JUnit
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-We are aware that there are newer testing tools such as TestNG, but we use `JUnit <http://junit.org>`_ because it's tried and true.
+We are aware that there are newer testing tools such as TestNG, but we use `JUnit <https://junit.org>`_ because it's tried and true.
 We support both (legacy) JUnit 4.x tests (forming the majority of our tests) and
 newer JUnit 5 based testing.
 
@@ -238,11 +238,11 @@ Remember, it’s only a test (and it's not graded)! Some guidelines to bear in m
 - Map out which logical functions you want to test
 - Understand what’s being tested and ensure it’s repeatable
 - Assert the conditions of success / return values for each operation
-  * A useful resource would be `HTTP status codes <http://www.restapitutorial.com/httpstatuscodes.html>`_
+  * A useful resource would be `HTTP status codes <https://www.restapitutorial.com/httpstatuscodes.html>`_
 - Let the code do the labor; automate everything that happens when you run your test file.
 - Just as with any development, if you’re stuck: ask for help!
 
-To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven <http://maven.apache.org/ref/3.1.0/maven-embedder/cli.html>`_. You should have Maven installed as per the `Development Environment <http://guides.dataverse.org/en/latest/developers/dev-environment.html>`_ guide, but if not it’s easily done via Homebrew: ``brew install maven``.
+To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven <https://maven.apache.org/ref/3.1.0/maven-embedder/cli.html>`_. You should have Maven installed as per the `Development Environment <https://guides.dataverse.org/en/latest/developers/dev-environment.html>`_ guide, but if not it’s easily done via Homebrew: ``brew install maven``.
 
 Once installed, you may run commands with ``mvn [options] [<goal(s)>] [<phase(s)>]``.
 
@@ -487,7 +487,7 @@ Future Work on Integration Tests
 - Automate testing of dataverse-client-python: https://github.com/IQSS/dataverse-client-python/issues/10
 - Work with @leeper on testing the R client: https://github.com/IQSS/dataverse-client-r
 - Review and attempt to implement "API Test Checklist" from @kcondon at https://docs.google.com/document/d/199Oq1YwQ4pYCguaeW48bIN28QAitSk63NbPYxJHCCAE/edit?usp=sharing
-- Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and http://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/
+- Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and https://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/
 - Consistent logging of API Tests. Show test name at the beginning and end and status codes returned.
 - expected passing and known/expected failing integration tests: https://github.com/IQSS/dataverse/issues/4438
 
diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst
index cbd27d6e8d2..17673ae499e 100755
--- a/doc/sphinx-guides/source/developers/tools.rst
+++ b/doc/sphinx-guides/source/developers/tools.rst
@@ -43,20 +43,20 @@ On Windows if you see an error like ``/usr/bin/perl^M: bad interpreter`` you mig
 PlantUML
 ++++++++
 
-PlantUML is used to create diagrams in the guides and other places. Download it from http://plantuml.com and check out an example script at https://github.com/IQSS/dataverse/blob/v4.6.1/doc/Architecture/components.sh . Note that for this script to work, you'll need the ``dot`` program, which can be installed on Mac with ``brew install graphviz``.
+PlantUML is used to create diagrams in the guides and other places. Download it from https://plantuml.com and check out an example script at https://github.com/IQSS/dataverse/blob/v4.6.1/doc/Architecture/components.sh . Note that for this script to work, you'll need the ``dot`` program, which can be installed on Mac with ``brew install graphviz``.
 
 Eclipse Memory Analyzer Tool (MAT)
 ++++++++++++++++++++++++++++++++++
 
 The Memory Analyzer Tool (MAT) from Eclipse can help you analyze heap dumps, showing you "leak suspects" such as seen at https://github.com/payara/Payara/issues/350#issuecomment-115262625
 
-It can be downloaded from http://www.eclipse.org/mat
+It can be downloaded from https://www.eclipse.org/mat
 
 If the heap dump provided to you was created with ``gcore`` (such as with ``gcore -o /tmp/app.core $app_pid``) rather than ``jmap``, you will need to convert the file before you can open it in MAT. Using ``app.core.13849`` as example of the original 33 GB file, here is how you could convert it into a 26 GB ``app.core.13849.hprof`` file. Please note that this operation took almost 90 minutes:
 
 ``/usr/java7/bin/jmap -dump:format=b,file=app.core.13849.hprof /usr/java7/bin/java app.core.13849``
 
-A file of this size may not "just work" in MAT. When you attempt to open it you may see something like "An internal error occurred during: "Parsing heap dump from '/tmp/heapdumps/app.core.13849.hprof'". Java heap space". If so, you will need to increase the memory allocated to MAT. On Mac OS X, this can be done by editing ``MemoryAnalyzer.app/Contents/MacOS/MemoryAnalyzer.ini`` and increasing the value "-Xmx1024m" until it's high enough to open the file. See also http://wiki.eclipse.org/index.php/MemoryAnalyzer/FAQ#Out_of_Memory_Error_while_Running_the_Memory_Analyzer
+A file of this size may not "just work" in MAT. When you attempt to open it you may see something like "An internal error occurred during: "Parsing heap dump from '/tmp/heapdumps/app.core.13849.hprof'". Java heap space". If so, you will need to increase the memory allocated to MAT. On Mac OS X, this can be done by editing ``MemoryAnalyzer.app/Contents/MacOS/MemoryAnalyzer.ini`` and increasing the value "-Xmx1024m" until it's high enough to open the file. See also https://wiki.eclipse.org/index.php/MemoryAnalyzer/FAQ#Out_of_Memory_Error_while_Running_the_Memory_Analyzer
 
 PageKite
 ++++++++
@@ -73,7 +73,7 @@ The first time you run ``./pagekite.py`` a file at ``~/.pagekite.rc`` will be
 created. You can edit this file to configure PageKite to serve up port 8080
 (the default app server HTTP port) or the port of your choosing.
 
-According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as the Dataverse Project.
+According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets https://opensource.org/docs/definition.php such as the Dataverse Project.
 
 MSV
 +++
diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst
index 2423877348f..856de209e82 100644
--- a/doc/sphinx-guides/source/developers/unf/index.rst
+++ b/doc/sphinx-guides/source/developers/unf/index.rst
@@ -27,7 +27,7 @@ with Dataverse Software 2.0 and throughout the 3.* lifecycle, UNF v.5
 UNF v.6. Two parallel implementation, in R and Java, will be
 available, for cross-validation.
 
-Learn more: Micah Altman and Gary King. 2007. “A Proposed Standard for the Scholarly Citation of Quantitative Data.� D-Lib Magazine, 13. Publisher’s Version Copy at http://j.mp/2ovSzoT
+Learn more: Micah Altman and Gary King. 2007. “A Proposed Standard for the Scholarly Citation of Quantitative Data.� D-Lib Magazine, 13. Publisher’s Version Copy at https://j.mp/2ovSzoT
 
 **Contents:**
 
diff --git a/doc/sphinx-guides/source/developers/unf/unf-v3.rst b/doc/sphinx-guides/source/developers/unf/unf-v3.rst
index 3f0018d7fa5..98c07b398e0 100644
--- a/doc/sphinx-guides/source/developers/unf/unf-v3.rst
+++ b/doc/sphinx-guides/source/developers/unf/unf-v3.rst
@@ -34,11 +34,11 @@ For example, the number pi at five digits is represented as -3.1415e+, and the n
 
 1. Terminate character strings representing nonmissing values with a POSIX end-of-line character.
 
-2. Encode each character string with `Unicode bit encoding <http://www.unicode.org/versions/Unicode4.0.0/>`_. Versions 3 through 4 use UTF-32BE; Version 4.1 uses UTF-8.
+2. Encode each character string with `Unicode bit encoding <https://www.unicode.org/versions/Unicode4.0.0/>`_. Versions 3 through 4 use UTF-32BE; Version 4.1 uses UTF-8.
 
 3. Combine the vector of character strings into a single sequence, with each character string separated by a POSIX end-of-line character and a null byte.
 
-4. Compute a hash on the resulting sequence using the standard MD5 hashing algorithm for Version 3 and using `SHA256 <http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf>`_ for Version 4. The resulting hash is `base64 <http://www.ietf.org/rfc/rfc3548.txt>`_ encoded to support readability.
+4. Compute a hash on the resulting sequence using the standard MD5 hashing algorithm for Version 3 and using `SHA256 <https://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf>`_ for Version 4. The resulting hash is `base64 <https://www.ietf.org/rfc/rfc3548.txt>`_ encoded to support readability.
 
 5. Calculate the UNF for each lower-level data object, using a consistent UNF version and level of precision across the individual UNFs being combined.
 
@@ -49,4 +49,4 @@ For example, the number pi at five digits is represented as -3.1415e+, and the n
 8. Combine UNFs from multiple variables to form a single UNF for an entire data frame, and then combine UNFs for a set of data frames to form a single UNF that represents an entire research study.
 
 Learn more: 
-Software for computing UNFs is available in an R Module, which includes a Windows standalone tool and code for Stata and SAS languages. Also see the following for more details: Micah Altman and Gary King. 2007. "A Proposed Standard for the Scholarly Citation of Quantitative Data," D-Lib Magazine, Vol. 13, No. 3/4 (March). (Abstract: `HTML <http://gking.harvard.edu/files/abs/cite-abs.shtml>`_ | Article: `PDF <http://gking.harvard.edu/files/cite.pdf>`_)
+Software for computing UNFs is available in an R Module, which includes a Windows standalone tool and code for Stata and SAS languages. Also see the following for more details: Micah Altman and Gary King. 2007. "A Proposed Standard for the Scholarly Citation of Quantitative Data," D-Lib Magazine, Vol. 13, No. 3/4 (March). (Abstract: `HTML <https://gking.harvard.edu/files/abs/cite-abs.shtml>`_ | Article: `PDF <https://gking.harvard.edu/files/cite.pdf>`_)
diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst
index 9648bae47c8..b2495ff3dd9 100644
--- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst
+++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst
@@ -156,7 +156,7 @@ For example, to specify a non-default precision the parameter it is specified us
 | Allowed values are {``128`` , ``192`` , ``196`` , ``256``} with ``128`` being the default. 
 | ``R1`` - **truncate** numeric values to ``N`` digits, **instead of rounding**, as previously described.
 
-`Dr. Micah Altman's classic UNF v5 paper <http://www.researchgate.net/publication/200043172_A_Fingerprint_Method_for_Scientific_Data_Verification>`_ mentions another optional parameter ``T###``, for specifying rounding of date and time values (implemented as stripping the values of entire components - fractional seconds, seconds, minutes, hours... etc., progressively) - but it doesn't specify its syntax. It is left as an exercise for a curious reader to contact the author and work out the details, if so desired. (Not implemented in UNF Version 6 by the Dataverse Project).
+`Dr. Micah Altman's classic UNF v5 paper <https://www.researchgate.net/publication/200043172_A_Fingerprint_Method_for_Scientific_Data_Verification>`_ mentions another optional parameter ``T###``, for specifying rounding of date and time values (implemented as stripping the values of entire components - fractional seconds, seconds, minutes, hours... etc., progressively) - but it doesn't specify its syntax. It is left as an exercise for a curious reader to contact the author and work out the details, if so desired. (Not implemented in UNF Version 6 by the Dataverse Project).
 
 Note: we do not recommend truncating character strings at fewer bytes than the default ``128`` (the ``X`` parameter). At the very least this number **must** be high enough so that the printable UNFs of individual variables or files are not truncated, when calculating combined UNFs of files or datasets, respectively. 
 
diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst
index aacc245af5a..31fc0a4e602 100644
--- a/doc/sphinx-guides/source/developers/version-control.rst
+++ b/doc/sphinx-guides/source/developers/version-control.rst
@@ -24,7 +24,7 @@ The goals of the Dataverse Software branching strategy are:
 - allow for concurrent development
 - only ship stable code
 
-We follow a simplified "git flow" model described at http://nvie.com/posts/a-successful-git-branching-model/ involving a "master" branch, a "develop" branch, and feature branches such as "1234-bug-fix".
+We follow a simplified "git flow" model described at https://nvie.com/posts/a-successful-git-branching-model/ involving a "master" branch, a "develop" branch, and feature branches such as "1234-bug-fix".
 
 Branches
 ~~~~~~~~
diff --git a/doc/sphinx-guides/source/index.rst b/doc/sphinx-guides/source/index.rst
index f7e81756e5b..37bb2353ff7 100755
--- a/doc/sphinx-guides/source/index.rst
+++ b/doc/sphinx-guides/source/index.rst
@@ -42,7 +42,7 @@ Other Resources
 Additional information about the Dataverse Project itself
 including presentations, information about upcoming releases, data
 management and citation, and announcements can be found at
-`http://dataverse.org/ <http://dataverse.org/>`__
+`https://dataverse.org/ <https://dataverse.org/>`__
 
 **User Group**
 
@@ -65,7 +65,7 @@ The support email address is `support@dataverse.org <mailto:support@dataverse.or
 **Reporting Issues and Contributing**
 
 Report bugs and add feature requests in `GitHub Issues <https://github.com/IQSS/dataverse/issues>`__
-or use `GitHub pull requests <http://guides.dataverse.org/en/latest/developers/version-control.html#how-to-make-a-pull-request>`__,
+or use `GitHub pull requests <https://guides.dataverse.org/en/latest/developers/version-control.html#how-to-make-a-pull-request>`__,
 if you have some code, scripts or documentation that you'd like to share.
 If you have a **security issue** to report, please email `security@dataverse.org <mailto:security@dataverse.org>`__.
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f2de9d5702f..0edb09784e1 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -112,7 +112,7 @@ The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has al
 
 Your decision to proxy or not should primarily be driven by which features of the Dataverse Software you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section.
 
-Even if you have no interest in Shibboleth, you may want to front your Dataverse installation with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse Project team <https://github.com/IQSS/dataverse/blob/v4.6.1/doc/shib/shib.md>`_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse Project team cannot provide much help with adding certificates to Payara beyond linking to `tips <http://stackoverflow.com/questions/906402/importing-an-existing-x509-certificate-and-private-key-in-java-keystore-to-use-i>`_ on the web.
+Even if you have no interest in Shibboleth, you may want to front your Dataverse installation with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse Project team <https://github.com/IQSS/dataverse/blob/v4.6.1/doc/shib/shib.md>`_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse Project team cannot provide much help with adding certificates to Payara beyond linking to `tips <https://stackoverflow.com/questions/906402/importing-an-existing-x509-certificate-and-private-key-in-java-keystore-to-use-i>`_ on the web.
 
 Still not convinced you should put Payara behind another web server? Even if you manage to get your SSL certificate into Payara, how are you going to run Payara on low ports such as 80 and 443? Are you going to run Payara as root? Bad idea. This is a security risk. Under "Additional Recommendations" under "Securing Your Installation" above you are advised to configure Payara to run as a user other than root.
 
@@ -124,7 +124,7 @@ If you really don't want to front Payara with any proxy (not recommended), you c
 
 ``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-2.port=443``
 
-What about port 80? Even if you don't front your Dataverse installation with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to
+What about port 80? Even if you don't front your Dataverse installation with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: https://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to
 
 If you are running an installation with Apache and Payara on the same server, and would like to restrict Payara from responding to any requests to port 8080 from external hosts (in other words, not through Apache), you can restrict the AJP listener to localhost only with:
 
@@ -157,7 +157,7 @@ and restart Payara. The prefix can be configured via the API (where it is referr
 
 Once this is done, you will be able to publish datasets and files, but the persistent identifiers will not be citable, and they will only resolve from the DataCite test environment (and then only if the Dataverse installation from which you published them is accessible - DOIs minted from your laptop will not resolve). Note that any datasets or files created using the test configuration cannot be directly migrated and would need to be created again once a valid DOI namespace is configured.
 
-To properly configure persistent identifiers for a production installation, an account and associated namespace must be acquired for a fee from a DOI or HDL provider. **DataCite** (https://www.datacite.org) is the recommended DOI provider (see https://dataversecommunity.global for more on joining DataCite) but **EZID** (http://ezid.cdlib.org) is an option for the University of California according to https://www.cdlib.org/cdlinfo/2017/08/04/ezid-doi-service-is-evolving/ . **Handle.Net** (https://www.handle.net) is the HDL provider.
+To properly configure persistent identifiers for a production installation, an account and associated namespace must be acquired for a fee from a DOI or HDL provider. **DataCite** (https://www.datacite.org) is the recommended DOI provider (see https://dataversecommunity.global for more on joining DataCite) but **EZID** (https://ezid.cdlib.org) is an option for the University of California according to https://www.cdlib.org/cdlinfo/2017/08/04/ezid-doi-service-is-evolving/ . **Handle.Net** (https://www.handle.net) is the HDL provider.
 
 Once you have your DOI or Handle account credentials and a namespace, configure your Dataverse installation to use them using the JVM options and database settings below.
 
@@ -205,7 +205,7 @@ Here are the configuration options for handles:
 - :ref:`:IndependentHandleService <:IndependentHandleService>` (optional)
 - :ref:`:HandleAuthHandle <:HandleAuthHandle>` (optional)
 
-Note: If you are **minting your own handles** and plan to set up your own handle service, please refer to `Handle.Net documentation <http://handle.net/hnr_documentation.html>`_.
+Note: If you are **minting your own handles** and plan to set up your own handle service, please refer to `Handle.Net documentation <https://handle.net/hnr_documentation.html>`_.
 
 .. _auth-modes:
 
@@ -288,7 +288,7 @@ Multiple file stores should specify different directories (which would nominally
 Swift Storage
 +++++++++++++
 
-Rather than storing data files on the filesystem, you can opt for an experimental setup with a `Swift Object Storage <http://swift.openstack.org>`_ backend. Each dataset that users create gets a corresponding "container" on the Swift side, and each data file is saved as a file within that container.
+Rather than storing data files on the filesystem, you can opt for an experimental setup with a `Swift Object Storage <https://swift.openstack.org>`_ backend. Each dataset that users create gets a corresponding "container" on the Swift side, and each data file is saved as a file within that container.
 
 **In order to configure a Swift installation,** you need to complete these steps to properly modify the JVM options:
 
@@ -304,7 +304,7 @@ First, run all the following create commands with your Swift endpoint informatio
     ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.<id>.username.endpoint1=your-username"
     ./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.<id>.endpoint.endpoint1=your-swift-endpoint"
 
-``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``http://rdgw.swift.example.org/swift/v1``.
+``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``https://rdgw.swift.example.org/swift/v1``.
 
 Then create a password alias by running (without changes):
 
@@ -400,7 +400,7 @@ You'll need an AWS account with an associated S3 bucket for your installation to
 **Make note** of the **bucket's name** and the **region** its data is hosted in.
 
 To **create a user** with full S3 access and nothing more for security reasons, we recommend using IAM
-(Identity and Access Management). See `IAM User Guide <http://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html>`_
+(Identity and Access Management). See `IAM User Guide <https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html>`_
 for more info on this process.
 
 **Generate the user keys** needed for a Dataverse installation afterwards by clicking on the created user.
@@ -410,7 +410,7 @@ for more info on this process.
   If you are hosting your Dataverse installation on an AWS EC2 instance alongside storage in S3, it is possible to use IAM Roles instead
   of the credentials file (the file at ``~/.aws/credentials`` mentioned below). Please note that you will still need the
   ``~/.aws/config`` file to specify the region. For more information on this option, see
-  http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+  https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
 
 Preparation When Using Custom S3-Compatible Service
 ###################################################
@@ -471,7 +471,7 @@ Additional profiles can be added to these files by appending the relevant inform
   aws_access_key_id = <insert key, no brackets>
   aws_secret_access_key = <insert secret key, no brackets>
 
-Place these two files in a folder named ``.aws`` under the home directory for the user running your Dataverse Installation on Payara. (From the `AWS Command Line Interface Documentation <http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html>`_:
+Place these two files in a folder named ``.aws`` under the home directory for the user running your Dataverse Installation on Payara. (From the `AWS Command Line Interface Documentation <https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html>`_:
 "In order to separate credentials from less sensitive options, region and output format are stored in a separate file
 named config in the same folder")
 
@@ -598,7 +598,7 @@ You may provide the values for these via any of the
 Reported Working S3-Compatible Storage
 ######################################
 
-`Minio v2018-09-12 <http://minio.io>`_
+`Minio v2018-09-12 <https://minio.io>`_
   Set ``dataverse.files.<id>.path-style-access=true``, as Minio works path-based. Works pretty smooth, easy to setup.
   **Can be used for quick testing, too:** just use the example values above. Uses the public (read: unsecure and
   possibly slow) https://play.minio.io:9000 service.
@@ -2063,7 +2063,7 @@ Note: by default, the URL is composed from the settings ``:GuidesBaseUrl`` and `
 :GuidesBaseUrl
 ++++++++++++++
 
-Set ``:GuidesBaseUrl`` to override the default value "http://guides.dataverse.org". If you are interested in writing your own version of the guides, you may find the :doc:`/developers/documentation` section of the Developer Guide helpful.
+Set ``:GuidesBaseUrl`` to override the default value "https://guides.dataverse.org". If you are interested in writing your own version of the guides, you may find the :doc:`/developers/documentation` section of the Developer Guide helpful.
 
 ``curl -X PUT -d http://dataverse.example.edu http://localhost:8080/api/admin/settings/:GuidesBaseUrl``
 
@@ -2084,14 +2084,14 @@ Set ``:NavbarSupportUrl`` to a fully-qualified URL which will be used for the "S
 
 Note that this will override the default behaviour for the "Support" menu option, which is to display the Dataverse collection 'feedback' dialog.
 
-``curl -X PUT -d http://dataverse.example.edu/supportpage.html http://localhost:8080/api/admin/settings/:NavbarSupportUrl``
+``curl -X PUT -d https://dataverse.example.edu/supportpage.html http://localhost:8080/api/admin/settings/:NavbarSupportUrl``
 
 :MetricsUrl
 +++++++++++
 
 Make the metrics component on the root Dataverse collection a clickable link to a website where you present metrics on your Dataverse installation, perhaps one of the community-supported tools mentioned in the :doc:`/admin/reporting-tools-and-queries` section of the Admin Guide.
 
-``curl -X PUT -d http://metrics.dataverse.example.edu http://localhost:8080/api/admin/settings/:MetricsUrl``
+``curl -X PUT -d https://metrics.dataverse.example.edu http://localhost:8080/api/admin/settings/:MetricsUrl``
 
 .. _:MaxFileUploadSizeInBytes:
 
diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst
index 4b000f1ef9e..5cb6e7153d4 100755
--- a/doc/sphinx-guides/source/installation/installation-main.rst
+++ b/doc/sphinx-guides/source/installation/installation-main.rst
@@ -98,7 +98,7 @@ The supplied site URL will be saved under the JVM option :ref:`dataverse.siteUrl
 
 The Dataverse Software uses JHOVE_ to help identify the file format (CSV, PNG, etc.) for files that users have uploaded. The installer places files called ``jhove.conf`` and ``jhoveConfig.xsd`` into the directory ``/usr/local/payara5/glassfish/domains/domain1/config`` by default and makes adjustments to the jhove.conf file based on the directory into which you chose to install Payara.
 
-.. _JHOVE: http://jhove.openpreservation.org
+.. _JHOVE: https://jhove.openpreservation.org
 
 Logging In
 ----------
@@ -118,7 +118,7 @@ Use the following credentials to log in:
 - username: dataverseAdmin
 - password: admin
 
-Congratulations! You have a working Dataverse installation. Soon you'll be tweeting at `@dataverseorg <https://twitter.com/dataverseorg>`_ asking to be added to the map at http://dataverse.org :)
+Congratulations! You have a working Dataverse installation. Soon you'll be tweeting at `@dataverseorg <https://twitter.com/dataverseorg>`_ asking to be added to the map at https://dataverse.org :)
 
 Trouble? See if you find an answer in the troubleshooting section below.
 
@@ -197,7 +197,7 @@ Be sure you save the changes made here and then restart your Payara server to te
 UnknownHostException While Deploying
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you are seeing "Caused by: java.net.UnknownHostException: myhost: Name or service not known" in server.log and your hostname is "myhost" the problem is likely that "myhost" doesn't appear in ``/etc/hosts``. See also http://stackoverflow.com/questions/21817809/glassfish-exception-during-deployment-project-with-stateful-ejb/21850873#21850873
+If you are seeing "Caused by: java.net.UnknownHostException: myhost: Name or service not known" in server.log and your hostname is "myhost" the problem is likely that "myhost" doesn't appear in ``/etc/hosts``. See also https://stackoverflow.com/questions/21817809/glassfish-exception-during-deployment-project-with-stateful-ejb/21850873#21850873
 
 .. _fresh-reinstall:
 
diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst
index 2251af7b81b..e5b10883d4b 100644
--- a/doc/sphinx-guides/source/installation/intro.rst
+++ b/doc/sphinx-guides/source/installation/intro.rst
@@ -2,7 +2,7 @@
 Introduction
 ============
 
-Welcome! Thanks for installing `The Dataverse Project <http://dataverse.org>`_!
+Welcome! Thanks for installing `The Dataverse Project <https://dataverse.org>`_!
 
 .. contents:: |toctitle|
 	:local:
@@ -36,7 +36,7 @@ Getting Help
 To get help installing or configuring a Dataverse installation, please try one or more of:
 
 - posting to the `dataverse-community <https://groups.google.com/forum/#!forum/dataverse-community>`_ Google Group.
-- asking at http://chat.dataverse.org
+- asking at https://chat.dataverse.org
 - emailing support@dataverse.org to open a private ticket at https://help.hmdc.harvard.edu
 
 Information to Send to Support When Installation Fails
diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst
index 0dfdb0393e0..cd765c91b7f 100644
--- a/doc/sphinx-guides/source/installation/oauth2.rst
+++ b/doc/sphinx-guides/source/installation/oauth2.rst
@@ -11,7 +11,7 @@ As explained under "Auth Modes" in the :doc:`config` section, OAuth2 is one of t
 
 `OAuth2 <https://oauth.net/2/>`_ is an authentication protocol that allows systems to share user data, while letting the users control what data is being shared. When you see buttons stating "login with Google" or "login through Facebook", OAuth2 is probably involved. For the purposes of this section, we will shorten "OAuth2" to just "OAuth." OAuth can be compared and contrasted with :doc:`shibboleth`.
 
-The Dataverse Software supports four OAuth providers: `ORCID <http://orcid.org>`_, `Microsoft Azure Active Directory (AD) <https://docs.microsoft.com/azure/active-directory/>`_, `GitHub <https://github.com>`_, and `Google <https://console.developers.google.com>`_.
+The Dataverse Software supports four OAuth providers: `ORCID <https://orcid.org>`_, `Microsoft Azure Active Directory (AD) <https://docs.microsoft.com/azure/active-directory/>`_, `GitHub <https://github.com>`_, and `Google <https://console.developers.google.com>`_.
 
 In addition :doc:`oidc` are supported, using a standard based on OAuth2.
 
diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index a40ef758dc7..ee154ca9b9c 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -51,7 +51,7 @@ Just like with :doc:`oauth2` you need to obtain a *Client ID* and a *Client Secr
   You need to apply for credentials out-of-band.
 
 The Dataverse installation will discover all necessary metadata for a given provider on its own (this is `part of the standard
-<http://openid.net/specs/openid-connect-discovery-1_0.html>`_).
+<https://openid.net/specs/openid-connect-discovery-1_0.html>`_).
 
 To enable this, you need to specify an *Issuer URL* when creating the configuration for your provider (see below).
 
diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst
index 3cf876a2251..7d458bbc37b 100644
--- a/doc/sphinx-guides/source/installation/prerequisites.rst
+++ b/doc/sphinx-guides/source/installation/prerequisites.rst
@@ -26,7 +26,7 @@ Installing Java
 
 The Dataverse Software should run fine with only the Java Runtime Environment (JRE) installed, but installing the Java Development Kit (JDK) is recommended so that useful tools for troubleshooting production environments are available. We recommend using Oracle JDK or OpenJDK.
 
-The Oracle JDK can be downloaded from http://www.oracle.com/technetwork/java/javase/downloads/index.html
+The Oracle JDK can be downloaded from https://www.oracle.com/technetwork/java/javase/downloads/index.html
 
 On a RHEL/derivative, install OpenJDK (devel version) using yum::
 
@@ -261,7 +261,7 @@ Installing jq
 or you may install it manually::
 
         # cd /usr/bin
-        # wget http://stedolan.github.io/jq/download/linux64/jq
+        # wget https://stedolan.github.io/jq/download/linux64/jq
         # chmod +x jq
         # jq --version
 
diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst
index cd0fbda77a6..3a2e1b99c70 100644
--- a/doc/sphinx-guides/source/installation/shibboleth.rst
+++ b/doc/sphinx-guides/source/installation/shibboleth.rst
@@ -76,7 +76,7 @@ A ``jk-connector`` network listener should have already been set up when you ran
 
 You can verify this with ``./asadmin list-network-listeners``. 
 
-This enables the `AJP protocol <http://en.wikipedia.org/wiki/Apache_JServ_Protocol>`_ used in Apache configuration files below.
+This enables the `AJP protocol <https://en.wikipedia.org/wiki/Apache_JServ_Protocol>`_ used in Apache configuration files below.
 
 SSLEngine Warning Workaround
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -93,7 +93,7 @@ Configure Apache
 Enforce HTTPS
 ~~~~~~~~~~~~~
 
-To prevent attacks such as `FireSheep <http://en.wikipedia.org/wiki/Firesheep>`_, HTTPS should be enforced. https://wiki.apache.org/httpd/RewriteHTTPToHTTPS provides a good method. You **could** copy and paste that those "rewrite rule" lines into Apache's main config file at ``/etc/httpd/conf/httpd.conf`` but using Apache's "virtual hosts" feature is recommended so that you can leave the main configuration file alone and drop a host-specific file into place.
+To prevent attacks such as `FireSheep <https://en.wikipedia.org/wiki/Firesheep>`_, HTTPS should be enforced. https://wiki.apache.org/httpd/RewriteHTTPToHTTPS provides a good method. You **could** copy and paste that those "rewrite rule" lines into Apache's main config file at ``/etc/httpd/conf/httpd.conf`` but using Apache's "virtual hosts" feature is recommended so that you can leave the main configuration file alone and drop a host-specific file into place.
 
 Below is an example of how "rewrite rule" lines look within a ``VirtualHost`` block. Download a :download:`sample file <../_static/installation/files/etc/httpd/conf.d/dataverse.example.edu.conf>` , edit it to substitute your own hostname under ``ServerName``, and place it at ``/etc/httpd/conf.d/dataverse.example.edu.conf`` or a filename that matches your hostname. The file must be in ``/etc/httpd/conf.d`` and must end in ".conf" to be included in Apache's configuration.
 
@@ -235,7 +235,7 @@ Run semodule
 
 Silent is golden. No output is expected. This will place a file in ``/etc/selinux/targeted/modules/active/modules/shibboleth.pp`` and include "shibboleth" in the output of ``semodule -l``. See the ``semodule`` man page if you ever want to remove or disable the module you just added.
 
-Congrats! You've made the creator of http://stopdisablingselinux.com proud. :)
+Congrats! You've made the creator of https://stopdisablingselinux.com proud. :)
 
 Restart Apache and Shibboleth
 -----------------------------
diff --git a/doc/sphinx-guides/source/style/foundations.rst b/doc/sphinx-guides/source/style/foundations.rst
index 31e0c314a05..cc193666868 100755
--- a/doc/sphinx-guides/source/style/foundations.rst
+++ b/doc/sphinx-guides/source/style/foundations.rst
@@ -9,7 +9,7 @@ Foundation elements are the very basic building blocks to create a page in Datav
 Grid Layout
 ===========
 
-`Bootstrap <http://getbootstrap.com/css/#grid>`__ provides a responsive, fluid, 12-column grid system that we use to organize our page layouts.
+`Bootstrap <https://getbootstrap.com/css/#grid>`__ provides a responsive, fluid, 12-column grid system that we use to organize our page layouts.
 
 We use the fixed-width ``.container`` class which provides responsive widths (i.e. auto, 750px, 970px or 1170px) based on media queries for the page layout, with a series of rows and columns for the content.
 
@@ -42,7 +42,7 @@ The grid layout uses ``.col-sm-*`` classes for horizontal groups of columns, ins
 Typography
 ==========
 
-The typeface, text size, and line-height are set in the `Bootstrap CSS <http://getbootstrap.com/css/#type>`__. We use Bootstrap's global default ``font-size`` of **14px**, with a ``line-height`` of **1.428**, which is applied to the ``<body>`` and all paragraphs.
+The typeface, text size, and line-height are set in the `Bootstrap CSS <https://getbootstrap.com/css/#type>`__. We use Bootstrap's global default ``font-size`` of **14px**, with a ``line-height`` of **1.428**, which is applied to the ``<body>`` and all paragraphs.
 
 .. code-block:: css
 
@@ -57,7 +57,7 @@ The typeface, text size, and line-height are set in the `Bootstrap CSS <http://g
 Color Palette
 =============
 
-The default color palette is set in the `Bootstrap CSS <http://getbootstrap.com/css/#less-variables-colors>`__. It provides the background, border, text and link colors used across the application.
+The default color palette is set in the `Bootstrap CSS <https://getbootstrap.com/css/#less-variables-colors>`__. It provides the background, border, text and link colors used across the application.
 
 
 Brand Colors
@@ -138,7 +138,7 @@ We use our brand color, a custom burnt orange ``{color:#C55B28;}``, which is set
 Text Colors
 -----------
 
-Text color is the default setting from `Bootstrap CSS <http://getbootstrap.com/css/#less-variables-scaffolding>`__.
+Text color is the default setting from `Bootstrap CSS <https://getbootstrap.com/css/#less-variables-scaffolding>`__.
 
 .. code-block:: css
 
@@ -163,7 +163,7 @@ Text color is the default setting from `Bootstrap CSS <http://getbootstrap.com/c
 Link Colors
 -----------
 
-Link color is the default setting from `Bootstrap CSS <http://getbootstrap.com/css/#less-variables-links>`__. The hover state color is set to 15% darker.
+Link color is the default setting from `Bootstrap CSS <https://getbootstrap.com/css/#less-variables-links>`__. The hover state color is set to 15% darker.
 
 **Please note**, there is a CSS override issue with the link color due to the use of both a Bootstrap stylesheet and a PrimeFaces stylesheet in the UI. We've added CSS such as ``.ui-widget-content a {color: #428BCA;}`` to our stylesheet to keep the link color consistent.
 
@@ -204,7 +204,7 @@ Link color is the default setting from `Bootstrap CSS <http://getbootstrap.com/c
 Contextual Classes
 ------------------
 
-Contextual classes from `Bootstrap CSS <http://getbootstrap.com/css/#helper-classes>`__ can be used to style background and text colors. Semantic colors include various colors assigned to meaningful contextual values. We convey meaning through color with a handful of emphasis utility classes.
+Contextual classes from `Bootstrap CSS <https://getbootstrap.com/css/#helper-classes>`__ can be used to style background and text colors. Semantic colors include various colors assigned to meaningful contextual values. We convey meaning through color with a handful of emphasis utility classes.
 
 .. raw:: html
 
@@ -259,7 +259,7 @@ We use various icons across the application, which we get from Bootstrap, FontCu
 Bootstrap Glyphicons
 --------------------
 
-There are over 250 glyphs in font format from the Glyphicon Halflings set provided by `Bootstrap <http://getbootstrap.com/components/#glyphicons>`__. We utilize these mainly as icons inside of buttons and in message blocks.
+There are over 250 glyphs in font format from the Glyphicon Halflings set provided by `Bootstrap <https://getbootstrap.com/components/#glyphicons>`__. We utilize these mainly as icons inside of buttons and in message blocks.
 
 .. raw:: html
 
@@ -305,7 +305,7 @@ The :doc:`/developers/fontcustom` section of the Developer Guide explains how to
 Socicon Icon Font 
 -----------------
 
-We use `Socicon <http://www.socicon.com>`__ for our custom social icons. In the footer we use icons for Twitter and Github. In our Share feature, we also use custom social icons to allow users to select from a list of social media channels.
+We use `Socicon <https://www.socicon.com>`__ for our custom social icons. In the footer we use icons for Twitter and Github. In our Share feature, we also use custom social icons to allow users to select from a list of social media channels.
 
 .. raw:: html
 
diff --git a/doc/sphinx-guides/source/style/patterns.rst b/doc/sphinx-guides/source/style/patterns.rst
index e96f17dc2ec..c6602ffa26e 100644
--- a/doc/sphinx-guides/source/style/patterns.rst
+++ b/doc/sphinx-guides/source/style/patterns.rst
@@ -1,7 +1,7 @@
 Patterns
 ++++++++
 
-Patterns are what emerge when using the foundation elements together with basic objects like buttons and alerts, more complex Javascript components from `Bootstrap <http://getbootstrap.com/components/>`__ like tooltips and dropdowns, and AJAX components from `PrimeFaces <https://www.primefaces.org/showcase/>`__ like datatables and commandlinks.
+Patterns are what emerge when using the foundation elements together with basic objects like buttons and alerts, more complex Javascript components from `Bootstrap <https://getbootstrap.com/components/>`__ like tooltips and dropdowns, and AJAX components from `PrimeFaces <https://www.primefaces.org/showcase/>`__ like datatables and commandlinks.
 
 .. contents:: |toctitle|
   :local:
@@ -9,7 +9,7 @@ Patterns are what emerge when using the foundation elements together with basic
 Navbar
 ======
 
-The `Navbar component <http://getbootstrap.com/components/#navbar>`__ from Bootstrap spans the top of the application and contains the logo/branding, aligned to the left, plus search form and links, aligned to the right.
+The `Navbar component <https://getbootstrap.com/components/#navbar>`__ from Bootstrap spans the top of the application and contains the logo/branding, aligned to the left, plus search form and links, aligned to the right.
 
 When logged in, the account name is a dropdown menu, linking the user to account-specific content and the log out link.
 
@@ -74,7 +74,7 @@ When logged in, the account name is a dropdown menu, linking the user to account
 Breadcrumbs
 ===========
 
-The breadcrumbs are displayed under the header, and provide a trail of links for users to navigate the hierarchy of containing objects, from file to dataset to Dataverse collection. It utilizes a JSF `repeat component <http://docs.oracle.com/javaee/6/javaserverfaces/2.0/docs/pdldocs/facelets/ui/repeat.html>`_ to iterate through the breadcrumbs.
+The breadcrumbs are displayed under the header, and provide a trail of links for users to navigate the hierarchy of containing objects, from file to dataset to Dataverse collection. It utilizes a JSF `repeat component <https://docs.oracle.com/javaee/6/javaserverfaces/2.0/docs/pdldocs/facelets/ui/repeat.html>`_ to iterate through the breadcrumbs.
 
 .. raw:: html
 
@@ -108,7 +108,7 @@ The breadcrumbs are displayed under the header, and provide a trail of links for
 Tables
 ======
 
-Most tables use the `DataTable components <https://www.primefaces.org/showcase/ui/data/datatable/basic.xhtml>`__ from PrimeFaces and are styled using the `Tables component <http://getbootstrap.com/css/#tables>`__ from Bootstrap.
+Most tables use the `DataTable components <https://www.primefaces.org/showcase/ui/data/datatable/basic.xhtml>`__ from PrimeFaces and are styled using the `Tables component <https://getbootstrap.com/css/#tables>`__ from Bootstrap.
 
 .. raw:: html
 
@@ -187,7 +187,7 @@ Most tables use the `DataTable components <https://www.primefaces.org/showcase/u
 Forms
 =====
 
-Forms fulfill various functions across the site, but we try to style them consistently. We use the ``.form-horizontal`` layout, which uses ``.form-group`` to create a grid of rows for the labels and inputs. The consistent style of forms is maintained using the `Forms component <http://getbootstrap.com/css/#forms>`__ from Bootstrap. Form elements like the `InputText component <https://www.primefaces.org/showcase/ui/input/inputText.xhtml>`__ from PrimeFaces are kept looking clean and consistent across each page.
+Forms fulfill various functions across the site, but we try to style them consistently. We use the ``.form-horizontal`` layout, which uses ``.form-group`` to create a grid of rows for the labels and inputs. The consistent style of forms is maintained using the `Forms component <https://getbootstrap.com/css/#forms>`__ from Bootstrap. Form elements like the `InputText component <https://www.primefaces.org/showcase/ui/input/inputText.xhtml>`__ from PrimeFaces are kept looking clean and consistent across each page.
 
 .. raw:: html
 
@@ -289,7 +289,7 @@ Here are additional form elements that are common across many pages, including r
 Buttons
 =======
 
-There are various types of buttons for various actions, so we have many components to use, including the `CommandButton component <https://www.primefaces.org/showcase/ui/button/commandButton.xhtml>`__ and `CommandLink component <https://www.primefaces.org/showcase/ui/button/commandLink.xhtml>`__ from PrimeFaces, as well as the basic JSF `Link component <http://docs.oracle.com/javaee/6/javaserverfaces/2.0/docs/pdldocs/facelets/h/link.html>`__ and `OutputLink component <http://docs.oracle.com/javaee/6/javaserverfaces/2.0/docs/pdldocs/facelets/h/outputLink.html>`__. Those are styled using the `Buttons component <http://getbootstrap.com/css/#buttons>`__, `Button Groups component <http://getbootstrap.com/components/#btn-groups>`__ and `Buttons Dropdowns component <http://getbootstrap.com/components/#btn-dropdowns>`__ from Bootstrap.
+There are various types of buttons for various actions, so we have many components to use, including the `CommandButton component <https://www.primefaces.org/showcase/ui/button/commandButton.xhtml>`__ and `CommandLink component <https://www.primefaces.org/showcase/ui/button/commandLink.xhtml>`__ from PrimeFaces, as well as the basic JSF `Link component <https://docs.oracle.com/javaee/6/javaserverfaces/2.0/docs/pdldocs/facelets/h/link.html>`__ and `OutputLink component <https://docs.oracle.com/javaee/6/javaserverfaces/2.0/docs/pdldocs/facelets/h/outputLink.html>`__. Those are styled using the `Buttons component <https://getbootstrap.com/css/#buttons>`__, `Button Groups component <https://getbootstrap.com/components/#btn-groups>`__ and `Buttons Dropdowns component <https://getbootstrap.com/components/#btn-dropdowns>`__ from Bootstrap.
 
 Action Buttons
 --------------
@@ -668,7 +668,7 @@ Another variation of icon-only buttons uses the ``.btn-link`` style class from B
 Pagination
 ==========
 
-We use the `Pagination component <http://getbootstrap.com/components/#pagination>`__ from Bootstrap for paging through search results.
+We use the `Pagination component <https://getbootstrap.com/components/#pagination>`__ from Bootstrap for paging through search results.
 
 .. raw:: html
 
@@ -738,7 +738,7 @@ We use the `Pagination component <http://getbootstrap.com/components/#pagination
 Labels
 ======
 
-The `Labels component <http://getbootstrap.com/components/#labels>`__ from Bootstrap is used for publication status (DRAFT, In Review, Unpublished, Deaccessioned), and Dataset version, as well as Tabular Data Tags (Survey, Time Series, Panel, Event, Genomics, Network, Geospatial).
+The `Labels component <https://getbootstrap.com/components/#labels>`__ from Bootstrap is used for publication status (DRAFT, In Review, Unpublished, Deaccessioned), and Dataset version, as well as Tabular Data Tags (Survey, Time Series, Panel, Event, Genomics, Network, Geospatial).
 
 .. raw:: html
 
@@ -768,7 +768,7 @@ The `Labels component <http://getbootstrap.com/components/#labels>`__ from Boots
 Alerts
 ======
 
-For our help/information, success, warning, and error message blocks we use a custom built UI component based on the `Alerts component <http://getbootstrap.com/components/#alerts>`__ from Bootstrap.
+For our help/information, success, warning, and error message blocks we use a custom built UI component based on the `Alerts component <https://getbootstrap.com/components/#alerts>`__ from Bootstrap.
 
 .. raw:: html
 
@@ -859,9 +859,9 @@ Style classes can be added to ``p``, ``div``, ``span`` and other elements to add
 Images
 ======
 
-For images, we use the `GraphicImage  component <https://www.primefaces.org/showcase/ui/multimedia/graphicImage.xhtml>`__ from PrimeFaces, or the basic JSF `GraphicImage component <http://docs.oracle.com/javaee/6/javaserverfaces/2.1/docs/vdldocs/facelets/h/graphicImage.html>`__.
+For images, we use the `GraphicImage  component <https://www.primefaces.org/showcase/ui/multimedia/graphicImage.xhtml>`__ from PrimeFaces, or the basic JSF `GraphicImage component <https://docs.oracle.com/javaee/6/javaserverfaces/2.1/docs/vdldocs/facelets/h/graphicImage.html>`__.
 
-To display images in a responsive way, they are styled with ``.img-responsive``, an `Images CSS class <http://getbootstrap.com/css/#images>`__ from Bootstrap.
+To display images in a responsive way, they are styled with ``.img-responsive``, an `Images CSS class <https://getbootstrap.com/css/#images>`__ from Bootstrap.
 
 .. raw:: html
 
@@ -879,7 +879,7 @@ To display images in a responsive way, they are styled with ``.img-responsive``,
 Panels
 ======
 
-The most common of our containers, the `Panels component <http://getbootstrap.com/components/#panels>`__ from Bootstrap is used to add a border and padding around sections of content like metadata blocks. Displayed with a header and/or footer, it can also be used with the  `Collapse plugin <http://getbootstrap.com/javascript/#collapse>`__ from Bootstrap.
+The most common of our containers, the `Panels component <https://getbootstrap.com/components/#panels>`__ from Bootstrap is used to add a border and padding around sections of content like metadata blocks. Displayed with a header and/or footer, it can also be used with the  `Collapse plugin <https://getbootstrap.com/javascript/#collapse>`__ from Bootstrap.
 
 .. raw:: html
 
@@ -943,7 +943,7 @@ Tabs
 
 Tabs are used to provide content panes on a page that allow the user to view different sections of content without navigating to a different page.
 
-We use the `TabView component <https://www.primefaces.org/showcase/ui/panel/tabView.xhtml>`__ from PrimeFaces, which is styled using the `Tab component <http://getbootstrap.com/javascript/#tabs>`__ from Bootstrap.
+We use the `TabView component <https://www.primefaces.org/showcase/ui/panel/tabView.xhtml>`__ from PrimeFaces, which is styled using the `Tab component <https://getbootstrap.com/javascript/#tabs>`__ from Bootstrap.
 
 .. raw:: html
 
@@ -989,7 +989,7 @@ Modals are dialog prompts that act as popup overlays, but don't create a new bro
 
 Buttons usually provide the UI prompt. A user clicks the button, which then opens a `Dialog component <https://www.primefaces.org/showcase/ui/overlay/dialog/basic.xhtml>`__  or `Confirm Dialog component <https://www.primefaces.org/showcase/ui/overlay/confirmDialog.xhtml>`__  from PrimeFaces that displays the modal with the necessary information and actions to take.
 
-The modal is styled using the `Modal component <http://getbootstrap.com/javascript/#modals>`__ from Bootstrap, for a popup window that prompts a user for information, with overlay and a backdrop, then header, content, and buttons. We can use style classes from Bootstrap for large (``.bs-example-modal-lg``) and small (``.bs-example-modal-sm``) width options.
+The modal is styled using the `Modal component <https://getbootstrap.com/javascript/#modals>`__ from Bootstrap, for a popup window that prompts a user for information, with overlay and a backdrop, then header, content, and buttons. We can use style classes from Bootstrap for large (``.bs-example-modal-lg``) and small (``.bs-example-modal-sm``) width options.
 
 .. raw:: html
 
diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst
index 12cc54c7fde..792fad730cf 100755
--- a/doc/sphinx-guides/source/user/account.rst
+++ b/doc/sphinx-guides/source/user/account.rst
@@ -109,7 +109,7 @@ If you are leaving your institution and need to convert your Dataverse installat
 ORCID Log In
 ~~~~~~~~~~~~~
 
-You can set up your Dataverse installation account to allow you to log in using your ORCID credentials. ORCID® is an independent non-profit effort to provide an open registry of unique researcher identifiers and open services to link research activities and organizations to these identifiers. Learn more at `orcid.org <http://orcid.org>`_. 
+You can set up your Dataverse installation account to allow you to log in using your ORCID credentials. ORCID® is an independent non-profit effort to provide an open registry of unique researcher identifiers and open services to link research activities and organizations to these identifiers. Learn more at `orcid.org <https://orcid.org>`_. 
 
 Create a Dataverse installation account using ORCID
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/doc/sphinx-guides/source/user/appendix.rst b/doc/sphinx-guides/source/user/appendix.rst
index b05459b6aaf..ae0ec37aff3 100755
--- a/doc/sphinx-guides/source/user/appendix.rst
+++ b/doc/sphinx-guides/source/user/appendix.rst
@@ -22,13 +22,13 @@ Supported Metadata
 
 Detailed below are what metadata schemas we support for Citation and Domain Specific Metadata in the Dataverse Project:
 
-- `Citation Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=0>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/citation.tsv>`__): compliant with `DDI Lite <http://www.ddialliance.org/specification/ddi2.1/lite/index.html>`_, `DDI 2.5 Codebook <http://www.ddialliance.org/>`__, `DataCite 3.1 <http://schema.datacite.org/meta/kernel-3.1/doc/DataCite-MetadataKernel_v3.1.pdf>`__, and Dublin Core's `DCMI Metadata Terms <http://dublincore.org/documents/dcmi-terms/>`__ . Language field uses `ISO 639-1 <https://www.loc.gov/standards/iso639-2/php/English_list.php>`__ controlled vocabulary.
-- `Geospatial Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=4>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/geospatial.tsv>`__): compliant with DDI Lite, DDI 2.5 Codebook, DataCite, and Dublin Core. Country / Nation field uses `ISO 3166-1 <http://en.wikipedia.org/wiki/ISO_3166-1>`_ controlled vocabulary.
+- `Citation Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=0>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/citation.tsv>`__): compliant with `DDI Lite <https://www.ddialliance.org/specification/ddi2.1/lite/index.html>`_, `DDI 2.5 Codebook <https://www.ddialliance.org/>`__, `DataCite 3.1 <https://schema.datacite.org/meta/kernel-3.1/doc/DataCite-MetadataKernel_v3.1.pdf>`__, and Dublin Core's `DCMI Metadata Terms <https://dublincore.org/documents/dcmi-terms/>`__ . Language field uses `ISO 639-1 <https://www.loc.gov/standards/iso639-2/php/English_list.php>`__ controlled vocabulary.
+- `Geospatial Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=4>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/geospatial.tsv>`__): compliant with DDI Lite, DDI 2.5 Codebook, DataCite, and Dublin Core. Country / Nation field uses `ISO 3166-1 <https://en.wikipedia.org/wiki/ISO_3166-1>`_ controlled vocabulary.
 - `Social Science & Humanities Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=1>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/social_science.tsv>`__): compliant with DDI Lite, DDI 2.5 Codebook, and Dublin Core.
 - `Astronomy and Astrophysics Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=3>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/astrophysics.tsv>`__): These metadata elements can be mapped/exported to the International Virtual Observatory Alliance’s (IVOA) 
-  `VOResource Schema format <http://www.ivoa.net/documents/latest/RM.html>`__ and is based on 
-  `Virtual Observatory (VO) Discovery and Provenance Metadata <http://perma.cc/H5ZJ-4KKY>`__.
-- `Life Sciences Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=2>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/biomedical.tsv>`__): based on `ISA-Tab Specification <https://isa-specs.readthedocs.io/en/latest/isamodel.html>`__, along with controlled vocabulary from subsets of the `OBI Ontology <http://bioportal.bioontology.org/ontologies/OBI>`__ and the `NCBI Taxonomy for Organisms <http://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/>`__.
+  `VOResource Schema format <https://www.ivoa.net/documents/latest/RM.html>`__ and is based on 
+  `Virtual Observatory (VO) Discovery and Provenance Metadata <https://perma.cc/H5ZJ-4KKY>`__.
+- `Life Sciences Metadata <https://docs.google.com/spreadsheet/ccc?key=0AjeLxEN77UZodHFEWGpoa19ia3pldEFyVFR0aFVGa0E#gid=2>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/biomedical.tsv>`__): based on `ISA-Tab Specification <https://isa-specs.readthedocs.io/en/latest/isamodel.html>`__, along with controlled vocabulary from subsets of the `OBI Ontology <https://bioportal.bioontology.org/ontologies/OBI>`__ and the `NCBI Taxonomy for Organisms <https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/>`__.
 - `Journal Metadata <https://docs.google.com/spreadsheets/d/13HP-jI_cwLDHBetn9UKTREPJ_F4iHdAvhjmlvmYdSSw/edit#gid=8>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/journals.tsv>`__): based on the `Journal Archiving and Interchange Tag Set, version 1.2 <https://jats.nlm.nih.gov/archiving/tag-library/1.2/chapter/how-to-read.html>`__.
 
 Experimental Metadata
diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst
index 77a760ef838..a3637154050 100755
--- a/doc/sphinx-guides/source/user/dataset-management.rst
+++ b/doc/sphinx-guides/source/user/dataset-management.rst
@@ -192,7 +192,7 @@ Additional download options available for tabular data (found in the same drop-d
 - As tab-delimited data (with the variable names in the first row); 
 - The original file uploaded by the user; 
 - Saved as R data (if the original file was not in R format); 
-- Variable Metadata (as a `DDI Codebook <http://www.ddialliance.org/Specification/DDI-Codebook/>`_ XML file);
+- Variable Metadata (as a `DDI Codebook <https://www.ddialliance.org/Specification/DDI-Codebook/>`_ XML file);
 - Data File Citation (currently in either RIS, EndNote XML, or BibTeX format); 
 - All of the above, as a zipped bundle. 
 
@@ -297,7 +297,7 @@ You can also search for files within datasets that have been tagged as "Workflow
 Astronomy (FITS)
 ----------------
 
-Metadata found in the header section of `Flexible Image Transport System (FITS) files <http://fits.gsfc.nasa.gov/fits_primer.html>`_ are automatically extracted by the Dataverse Software, aggregated and displayed in the Astronomy Domain-Specific Metadata of the Dataset that the file belongs to. This FITS file metadata, is therefore searchable and browsable (facets) at the Dataset-level.
+Metadata found in the header section of `Flexible Image Transport System (FITS) files <https://fits.gsfc.nasa.gov/fits_primer.html>`_ are automatically extracted by the Dataverse Software, aggregated and displayed in the Astronomy Domain-Specific Metadata of the Dataset that the file belongs to. This FITS file metadata, is therefore searchable and browsable (facets) at the Dataset-level.
 
 Compressed Files
 ----------------
@@ -388,7 +388,7 @@ Choosing a License
 ------------------
 
 Each Dataverse installation provides a set of license(s) data can be released under, and whether users can specify custom terms instead (see below). 
-One of the available licenses (often the `Creative Commons CC0 Public Domain Dedication <http://creativecommons.org/publicdomain/zero/1.0>`_) serves as the default if you do not make an explicit choice.
+One of the available licenses (often the `Creative Commons CC0 Public Domain Dedication <https://creativecommons.org/publicdomain/zero/1.0>`_) serves as the default if you do not make an explicit choice.
 If you want to apply one of the other available licenses to your dataset, you can change it on the Terms tab of your Dataset page.
 
 License Selection and Professional Norms
diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst
index f1d5611ede9..33ae9b555e6 100644
--- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst
+++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst
@@ -27,7 +27,7 @@ separately, in a relational database, so that it can be accessed
 efficiently by the application. For the purposes of archival
 preservation it can be exported, in plain text XML files, using a
 standardized, open `DDI Codebook
-<http://www.ddialliance.org/Specification/DDI-Codebook/2.5/>`_
+<https://www.ddialliance.org/Specification/DDI-Codebook/2.5/>`_
 format. (more info below)
 
 
@@ -53,6 +53,6 @@ Tabular Metadata in the Dataverse Software
 
 The structure of the metadata defining tabular data variables used in
 the Dataverse Software was originally based on the `DDI Codebook
-<http://www.ddialliance.org/Specification/DDI-Codebook/2.5/>`_ format.
+<https://www.ddialliance.org/Specification/DDI-Codebook/2.5/>`_ format.
 
 You can see an example of DDI output under the :ref:`data-variable-metadata-access` section of the :doc:`/api/dataaccess` section of the API Guide.

From 044ed40c17e1fa5fdbb7c8745a4671add25414c0 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Tue, 18 Oct 2022 00:32:55 +0200
Subject: [PATCH 005/922] Align table boundary in SWORD doc

---
 doc/sphinx-guides/source/api/sword.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst
index c9ac83bc204..51391784bde 100755
--- a/doc/sphinx-guides/source/api/sword.rst
+++ b/doc/sphinx-guides/source/api/sword.rst
@@ -127,7 +127,7 @@ Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse Project DB Element Cr
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
 |dcterms:creator              |         authorName (LastName, FirstName)     |       Y      |  Author(s) for the Dataset.                                                                                                                                 |
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
-|dcterms:subject              |   subject (Controlled Vocabulary) OR keyword |       Y      |  Controlled Vocabulary list is in our User Guide > `Metadata References <https://guides.dataverse.org/en/latest/user/appendix.html#metadata-references>`_.   |                                                                                                                
+|dcterms:subject              |   subject (Controlled Vocabulary) OR keyword |       Y      |  Controlled Vocabulary list is in our User Guide > `Metadata References <https://guides.dataverse.org/en/latest/user/appendix.html#metadata-references>`_.  |                                                                                                                
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
 |dcterms:description          |              dsDescriptionValue              |       Y      |  Describing the purpose, scope or nature of the Dataset. Can also use dcterms:abstract.                                                                     |
 +-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+

From cbc42d5052f8a9afc30121082a44c128387e2023 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Tue, 8 Nov 2022 14:07:32 +0100
Subject: [PATCH 006/922] renamed and moved the direct upload JVM option in the
 documentation

---
 doc/release-notes/4.20-release-notes.md                    | 7 +------
 doc/sphinx-guides/source/installation/config.rst           | 7 ++++++-
 .../edu/harvard/iq/dataverse/dataaccess/StorageIO.java     | 2 +-
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java  | 2 +-
 .../java/edu/harvard/iq/dataverse/util/SystemConfig.java   | 2 +-
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/doc/release-notes/4.20-release-notes.md b/doc/release-notes/4.20-release-notes.md
index 79037d8cd8c..ec52b638274 100644
--- a/doc/release-notes/4.20-release-notes.md
+++ b/doc/release-notes/4.20-release-notes.md
@@ -90,14 +90,10 @@ Also note that the :MaxFileUploadSizeInBytes property has a new option to provid
 
 ### Direct S3 Upload Changes
 
-Direct upload to S3 in UI and API is enabled per store by one new jvm option:
+Direct upload to S3 is enabled per store by one new jvm option:
 
     ./asadmin create-jvm-options "\-Ddataverse.files.<id>.upload-redirect=true"
 
-This option makes direct upload the default in the UI. In the API, you can use either: direct upload or upload via Dataverse upload. Direct upload to S3 in API only is enabled per store by this new jvm option:
-
-    ./asadmin create-jvm-options "\-Ddataverse.files.<id>.api-direct-upload=true"
-
 That option leaves via Dataverse upload by default in UI, but makes both: uploads via Dataverse and direct uploads possible via API.
 
 The existing :MaxFileUploadSizeInBytes property and ```dataverse.files.<id>.url-expiration-minutes``` jvm option for the same store also apply to direct upload.
@@ -133,7 +129,6 @@ We made changes to the JSON Export in this release (Issue 6650, PR #6669). If yo
 
 - The JVM option dataverse.files.file.directory=<your directory> controls where temporary files are stored (in the /temp subdir of the defined directory), independent of the location of any 'file' store defined above.
 - The JVM option dataverse.files.<id>.upload-redirect enables direct upload of files added to a dataset in the S3 bucket. (S3 stores only!)
-- The JVM option dataverse.files.<id>.api-direct-upload enables direct upload of files added to a dataset in any storage. (Via API only and when the uploading tool has direct access to the relevant storage used; i.e., upload the file first and register it via API!)
 - The JVM option dataverse.files.<id>.MaxFileUploadSizeInBytes controls the maximum size of file uploads allowed for the given file store. 
 - The JVM option dataverse.files.<id>.ingestsizelimit controls the maximum size of files for which ingest will be attempted, for the given file store. 
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 2e68bfaa1ab..4f15ad81190 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -249,6 +249,12 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
+When using integration tools, dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.allow-out-of-band-upload`` JVM option to ``true``.
+Files can be then uploaded by an integration tool with ``datasets/{id}/add`` api call, or uploaded directly to the storage and registerd in a dataset afterwards using the ``datasets/{id}/addFiles`` api call.
+Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``allow-out-of-band-upload`` and will enable direct upload even with ``allow-out-of-band-upload`` not set (or set to false).
+In other words, ``dataverse.files.\<id\>.allow-out-of-band-upload`` option opens the ``datasets/{id}/add`` and ``datasets/{id}/addFiles`` api endpoints without redirecting uploads in the UI.
+Enabling the ``upload-redirect`` option allows then direct upload automatically, without the need of enabling the ``allow-out-of-band-upload`` (setting it to ``false`` does not have any effect in that case).
+
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 
 Multi-store Basics
@@ -546,7 +552,6 @@ List of S3 Storage Options
     dataverse.files.<id>.bucket-name             <?>                 The bucket name. See above.                                                 (none)
     dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download or proxy through Dataverse.                          ``false``
     dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset in the S3 store.           ``false``
-    dataverse.files.<id>.api-direct-upload       ``true``/``false``  Enable direct upload of files added to a dataset via API only.              ``false``
     dataverse.files.<id>.ingestsizelimit         <size in bytes>     Maximum size of directupload files that should be ingested                  (none)
     dataverse.files.<id>.url-expiration-minutes  <?>                 If direct uploads/downloads: time until links expire. Optional.             60
     dataverse.files.<id>.min-part-size           <?>                 Multipart direct uploads will occur for files larger than this. Optional.   ``1024**3``
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index 8e2dd9fa961..a2ff546ef0a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -606,7 +606,7 @@ public static String getDriverPrefix(String driverId) {
     
     public static boolean isDirectUploadEnabled(String driverId) {
         return Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".api-direct-upload"));
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".allow-out-of-band-upload"));
     }
     
     //Check that storageIdentifier is consistent with store's config
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 0a41da4f7dd..db82df72b8a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1749,7 +1749,7 @@ public static boolean isPackageFile(DataFile dataFile) {
     public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	String driverId = dataset.getEffectiveStorageDriverId();
     	boolean directEnabled = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".api-direct-upload"));
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".allow-out-of-band-upload"));
     	//Should only be requested when it is allowed, but we'll log a warning otherwise
     	if(!directEnabled) {
     		logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId());
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 4585d99a01f..b040f557895 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -1112,7 +1112,7 @@ public boolean isDatafileValidationOnPublishEnabled() {
     }
 
 	public boolean directUploadEnabled(DvObjectContainer container) {
-        // this method is used in UI only, therfore "dataverse.files." + driverId + ".api-direct-upload" is not used here
+        // this method is used in UI only, therfore "dataverse.files." + driverId + ".allow-out-of-band-upload" is not used here
     	return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
 	}
 	

From 4abac1ac15d77f2f059977254971cf4be0f3f1f1 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 09:40:34 +0100
Subject: [PATCH 007/922] revert by accident editted old release notes

---
 doc/release-notes/4.20-release-notes.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/doc/release-notes/4.20-release-notes.md b/doc/release-notes/4.20-release-notes.md
index ec52b638274..e29953db101 100644
--- a/doc/release-notes/4.20-release-notes.md
+++ b/doc/release-notes/4.20-release-notes.md
@@ -93,9 +93,7 @@ Also note that the :MaxFileUploadSizeInBytes property has a new option to provid
 Direct upload to S3 is enabled per store by one new jvm option:
 
     ./asadmin create-jvm-options "\-Ddataverse.files.<id>.upload-redirect=true"
-
-That option leaves via Dataverse upload by default in UI, but makes both: uploads via Dataverse and direct uploads possible via API.
-
+    
 The existing :MaxFileUploadSizeInBytes property and ```dataverse.files.<id>.url-expiration-minutes``` jvm option for the same store also apply to direct upload.
 
 Direct upload via the Dataverse web interface is transparent to the user and handled automatically by the browser. Some minor differences in file upload exist: directly uploaded files are not unzipped and Dataverse does not scan their content to help in assigning a MIME type. Ingest of tabular files and metadata extraction from FITS files will occur, but can be turned off for files above a specified size limit through the new dataverse.files.<id>.ingestsizelimit jvm option.
@@ -128,7 +126,7 @@ We made changes to the JSON Export in this release (Issue 6650, PR #6669). If yo
 ## New JVM Options for file storage drivers
 
 - The JVM option dataverse.files.file.directory=<your directory> controls where temporary files are stored (in the /temp subdir of the defined directory), independent of the location of any 'file' store defined above.
-- The JVM option dataverse.files.<id>.upload-redirect enables direct upload of files added to a dataset in the S3 bucket. (S3 stores only!)
+- The JVM option dataverse.files.<id>.upload-redirect enables direct upload of files added to a dataset to the S3 bucket. (S3 stores only!)
 - The JVM option dataverse.files.<id>.MaxFileUploadSizeInBytes controls the maximum size of file uploads allowed for the given file store. 
 - The JVM option dataverse.files.<id>.ingestsizelimit controls the maximum size of files for which ingest will be attempted, for the given file store. 
 

From 578c7af84e7cd1eac52901643d9bb49bc878cfa3 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 10:05:20 +0100
Subject: [PATCH 008/922] indentation fixes

---
 .../iq/dataverse/util/SystemConfig.java       | 407 +++++++++---------
 1 file changed, 205 insertions(+), 202 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index b040f557895..f3d8e46b004 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -59,8 +59,8 @@ public class SystemConfig {
 
     @EJB
     AuthenticationServiceBean authenticationService;
-    
-   public static final String DATAVERSE_PATH = "/dataverse/";
+
+    public static final String DATAVERSE_PATH = "/dataverse/";
 
     /**
      * A JVM option for the advertised fully qualified domain name (hostname) of
@@ -70,11 +70,11 @@ public class SystemConfig {
      * The equivalent in DVN 3.x was "dvn.inetAddress".
      */
     public static final String FQDN = "dataverse.fqdn";
-    
+
     /**
      * A JVM option for specifying the "official" URL of the site.
-     * Unlike the FQDN option above, this would be a complete URL, 
-     * with the protocol, port number etc. 
+     * Unlike the FQDN option above, this would be a complete URL,
+     * with the protocol, port number etc.
      */
     public static final String SITE_URL = "dataverse.siteUrl";
 
@@ -102,41 +102,41 @@ public class SystemConfig {
     private String saneDefaultForSolrHostColonPort = "localhost:8983";
 
     /**
-     * The default number of datafiles that we allow to be created through 
+     * The default number of datafiles that we allow to be created through
      * zip file upload.
      */
-    private static final int defaultZipUploadFilesLimit = 1000; 
+    private static final int defaultZipUploadFilesLimit = 1000;
     public static final long defaultZipDownloadLimit = 104857600L; // 100MB
     private static final int defaultMultipleUploadFilesLimit = 1000;
     private static final int defaultLoginSessionTimeout = 480; // = 8 hours
 
-    private static String appVersionString = null; 
-    private static String buildNumberString = null; 
-    
+    private static String appVersionString = null;
+    private static String buildNumberString = null;
+
     private static final String JVM_TIMER_SERVER_OPTION = "dataverse.timerServer";
-    
-    private static final long DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT = 5000L; 
+
+    private static final long DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT = 5000L;
     private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE = 3000000L; // 3 MB
     private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_PDF = 1000000L; // 1 MB
-    
+
     public final static String DEFAULTCURATIONLABELSET = "DEFAULT";
     public final static String CURATIONLABELSDISABLED = "DISABLED";
-    
+
     public String getVersion() {
         return getVersion(false);
     }
-    
+
     // The return value is a "prviate static String", that should be initialized
-    // once, on the first call (see the code below)... But this is a @Stateless 
-    // bean... so that would mean "once per thread"? - this would be a prime 
+    // once, on the first call (see the code below)... But this is a @Stateless
+    // bean... so that would mean "once per thread"? - this would be a prime
     // candidate for being moved into some kind of an application-scoped caching
     // service... some CachingService @Singleton - ? (L.A. 5.8)
     public String getVersion(boolean withBuildNumber) {
-        
+
         if (appVersionString == null) {
 
             // The Version Number is no longer supplied in a .properties file - so
-            // we can't just do 
+            // we can't just do
             //  return BundleUtil.getStringFromBundle("version.number", null, ResourceBundle.getBundle("VersionNumber", Locale.US));
             //
             // Instead, we'll rely on Maven placing the version number into the
@@ -144,31 +144,31 @@ public String getVersion(boolean withBuildNumber) {
             // (this is considered a better practice, and will also allow us
             // to maintain this number in only one place - the pom.xml file)
             // -- L.A. 4.0.2
-            
-            // One would assume, that once the version is in the MANIFEST.MF, 
-            // as Implementation-Version:, it would be possible to obtain 
-            // said version simply as 
+
+            // One would assume, that once the version is in the MANIFEST.MF,
+            // as Implementation-Version:, it would be possible to obtain
+            // said version simply as
             //    appVersionString = getClass().getPackage().getImplementationVersion();
-            // alas - that's not working, for whatever reason. (perhaps that's 
+            // alas - that's not working, for whatever reason. (perhaps that's
             // only how it works with jar-ed packages; not with .war files).
-            // People on the interwebs suggest that one should instead 
-            // open the Manifest as a resource, then extract its attributes. 
-            // There were some complications with that too. Plus, relying solely 
-            // on the MANIFEST.MF would NOT work for those of the developers who 
-            // are using "in place deployment" (i.e., where 
-            // Netbeans runs their builds directly from the local target 
-            // directory, bypassing the war file deployment; and the Manifest 
-            // is only available in the .war file). For that reason, I am 
-            // going to rely on the pom.properties file, and use java.util.Properties 
+            // People on the interwebs suggest that one should instead
+            // open the Manifest as a resource, then extract its attributes.
+            // There were some complications with that too. Plus, relying solely
+            // on the MANIFEST.MF would NOT work for those of the developers who
+            // are using "in place deployment" (i.e., where
+            // Netbeans runs their builds directly from the local target
+            // directory, bypassing the war file deployment; and the Manifest
+            // is only available in the .war file). For that reason, I am
+            // going to rely on the pom.properties file, and use java.util.Properties
             // to read it. We have to look for this file in 2 different places
-            // depending on whether this is a .war file deployment, or a 
+            // depending on whether this is a .war file deployment, or a
             // developers build. (the app-level META-INF is only populated when
-            // a .war file is built; the "maven-archiver" directory, on the other 
+            // a .war file is built; the "maven-archiver" directory, on the other
             // hand, is only available when it's a local build deployment).
-            // So, long story short, I'm resorting to the convoluted steps below. 
-            // It may look hacky, but it should actually be pretty solid and 
-            // reliable. 
-            
+            // So, long story short, I'm resorting to the convoluted steps below.
+            // It may look hacky, but it should actually be pretty solid and
+            // reliable.
+
             
             // First, find the absolute path url of the application persistence file
             // always supplied with the Dataverse app:
@@ -180,46 +180,46 @@ public String getVersion(boolean withBuildNumber) {
                 filePath = fileUrl.getFile();
                 if (filePath != null) {
                     InputStream mavenPropertiesInputStream = null;
-                    String mavenPropertiesFilePath; 
+                    String mavenPropertiesFilePath;
                     Properties mavenProperties = new Properties();
 
                     
                     filePath = filePath.replaceFirst("/[^/]*$", "/");
-                    // Using a relative path, find the location of the maven pom.properties file. 
-                    // First, try to look for it in the app-level META-INF. This will only be 
-                    // available if it's a war file deployment: 
+                    // Using a relative path, find the location of the maven pom.properties file.
+                    // First, try to look for it in the app-level META-INF. This will only be
+                    // available if it's a war file deployment:
                     mavenPropertiesFilePath = filePath.concat("../../../META-INF/maven/edu.harvard.iq/dataverse/pom.properties");                                     
-                    
+
                     try {
                         mavenPropertiesInputStream = new FileInputStream(mavenPropertiesFilePath);
                     } catch (IOException ioex) {
-                        // OK, let's hope this is a local dev. build. 
-                        // In that case the properties file should be available in 
-                        // the maven-archiver directory: 
-                        
+                        // OK, let's hope this is a local dev. build.
+                        // In that case the properties file should be available in
+                        // the maven-archiver directory:
+
                         mavenPropertiesFilePath = filePath.concat("../../../../maven-archiver/pom.properties");
-                        
-                        // try again: 
-                        
+
+                        // try again:
+
                         try {
                             mavenPropertiesInputStream = new FileInputStream(mavenPropertiesFilePath);
                         } catch (IOException ioex2) {
                             logger.warning("Failed to find and/or open for reading the pom.properties file.");
-                            mavenPropertiesInputStream = null; 
+                            mavenPropertiesInputStream = null;
                         }
                     }
-                    
+
                     if (mavenPropertiesInputStream != null) {
                         try {
                             mavenProperties.load(mavenPropertiesInputStream);
-                            appVersionString = mavenProperties.getProperty("version");                        
+                            appVersionString = mavenProperties.getProperty("version");
                         } catch (IOException ioex) {
                             logger.warning("caught IOException trying to read and parse the pom properties file.");
                         } finally {
                             IOUtils.closeQuietly(mavenPropertiesInputStream);
                         }
                     }
-                    
+
                 } else {
                     logger.warning("Null file path representation of the location of persistence.xml in the webapp root directory!"); 
                 }
@@ -229,53 +229,54 @@ public String getVersion(boolean withBuildNumber) {
 
             
             if (appVersionString == null) {
-                // still null? - defaulting to 4.0:    
+                // still null? - defaulting to 4.0:
                 appVersionString = "4.0";
             }
         }
-            
+
         if (withBuildNumber) {
             if (buildNumberString == null) {
-                // (build number is still in a .properties file in the source tree; it only 
-                // contains a real build number if this war file was built by 
-                // Jenkins) 
-                        
+                // (build number is still in a .properties file in the source tree; it only
+                // contains a real build number if this war file was built by
+                // Jenkins)
+
                 try {
                     buildNumberString = ResourceBundle.getBundle("BuildNumber").getString("build.number");
                 } catch (MissingResourceException ex) {
-                    buildNumberString = null; 
+                    buildNumberString = null;
                 }
             }
-            
+
             if (buildNumberString != null && !buildNumberString.equals("")) {
-                return appVersionString + " build " + buildNumberString; 
-            } 
-        }        
-        
-        return appVersionString; 
+                return appVersionString + " build " + buildNumberString;
+            }
+        }
+
+        return appVersionString;
     }
 
     public String getSolrHostColonPort() {
         String SolrHost;
         if ( System.getenv("SOLR_SERVICE_HOST") != null && System.getenv("SOLR_SERVICE_HOST") != ""){
             SolrHost = System.getenv("SOLR_SERVICE_HOST");
+        } else {
+            SolrHost = saneDefaultForSolrHostColonPort;
         }
-        else SolrHost = saneDefaultForSolrHostColonPort;
         String solrHostColonPort = settingsService.getValueForKey(SettingsServiceBean.Key.SolrHostColonPort, SolrHost);
         return solrHostColonPort;
     }
 
     public boolean isProvCollectionEnabled() {
         String provCollectionEnabled = settingsService.getValueForKey(SettingsServiceBean.Key.ProvCollectionEnabled, null);
-        if("true".equalsIgnoreCase(provCollectionEnabled)){         
+        if ("true".equalsIgnoreCase(provCollectionEnabled)) {
             return true;
         }
         return false;
 
     }
-    
+
     public int getMetricsCacheTimeoutMinutes() {
-        int defaultValue = 10080; //one week in minutes
+        int defaultValue = 10080; // one week in minutes
         SettingsServiceBean.Key key = SettingsServiceBean.Key.MetricsCacheTimeoutMinutes;
         String metricsCacheTimeString = settingsService.getValueForKey(key);
         if (metricsCacheTimeString != null) {
@@ -293,7 +294,7 @@ public int getMetricsCacheTimeoutMinutes() {
         }
         return defaultValue;
     }
-    
+
     public int getMinutesUntilConfirmEmailTokenExpires() {
         final int minutesInOneDay = 1440;
         final int reasonableDefault = minutesInOneDay;
@@ -338,10 +339,10 @@ public static int getMinutesUntilPasswordResetTokenExpires() {
         }
         return reasonableDefault;
     }
-    
+
     /**
      * The "official", designated URL of the site;
-     * can be defined as a complete URL; or derived from the 
+     * can be defined as a complete URL; or derived from the
      * "official" hostname. If none of these options is set,
      * defaults to the InetAddress.getLocalHOst() and https;
      * These are legacy JVM options. Will be eventualy replaced
@@ -350,7 +351,7 @@ public static int getMinutesUntilPasswordResetTokenExpires() {
     public String getDataverseSiteUrl() {
         return getDataverseSiteUrlStatic();
     }
-    
+
     public static String getDataverseSiteUrlStatic() {
         String hostUrl = System.getProperty(SITE_URL);
         if (hostUrl != null && !"".equals(hostUrl)) {
@@ -367,19 +368,20 @@ public static String getDataverseSiteUrlStatic() {
         hostUrl = "https://" + hostName;
         return hostUrl;
     }
-    
+
     /**
-     * URL Tracking: 
+     * URL Tracking:
      */
     public String getPageURLWithQueryString() {
-        return PrettyContext.getCurrentInstance().getRequestURL().toURL() + PrettyContext.getCurrentInstance().getRequestQueryString().toQueryString();
+        return PrettyContext.getCurrentInstance().getRequestURL().toURL()
+                + PrettyContext.getCurrentInstance().getRequestQueryString().toQueryString();
     }
 
     /**
-     * The "official" server's fully-qualified domain name: 
+     * The "official" server's fully-qualified domain name:
      */
     public String getDataverseServer() {
-        // still reliese on a JVM option: 
+        // still reliese on a JVM option:
         String fqdn = System.getProperty(FQDN);
         if (fqdn == null) {
             try {
@@ -447,44 +449,44 @@ public static int getIntLimitFromStringOrDefault(String limitSetting, Integer de
 
     /**
      * Download-as-zip size limit.
-     * returns defaultZipDownloadLimit if not specified; 
-     * set to -1 to disable zip downloads. 
+     * returns defaultZipDownloadLimit if not specified;
+     * set to -1 to disable zip downloads.
      */
     public long getZipDownloadLimit() {
         String zipLimitOption = settingsService.getValueForKey(SettingsServiceBean.Key.ZipDownloadLimit);
         return getLongLimitFromStringOrDefault(zipLimitOption, defaultZipDownloadLimit);
     }
-    
+
     public int getZipUploadFilesLimit() {
         String limitOption = settingsService.getValueForKey(SettingsServiceBean.Key.ZipUploadFilesLimit);
         return getIntLimitFromStringOrDefault(limitOption, defaultZipUploadFilesLimit);
     }
-    
+
     /**
-     * Session timeout, in minutes. 
+     * Session timeout, in minutes.
      * (default value provided)
      */
     public int getLoginSessionTimeout() {
         return getIntLimitFromStringOrDefault(
-                settingsService.getValueForKey(SettingsServiceBean.Key.LoginSessionTimeout), 
-                defaultLoginSessionTimeout); 
+                settingsService.getValueForKey(SettingsServiceBean.Key.LoginSessionTimeout),
+                defaultLoginSessionTimeout);
     }
-    
+
     /*
     `   the number of files the GUI user is allowed to upload in one batch, 
         via drag-and-drop, or through the file select dialog
-    */
+     */
     public int getMultipleUploadFilesLimit() {
         String limitOption = settingsService.getValueForKey(SettingsServiceBean.Key.MultipleUploadFilesLimit);
         return getIntLimitFromStringOrDefault(limitOption, defaultMultipleUploadFilesLimit);
     }
-    
+
     public long getGuestbookResponsesPageDisplayLimit() {
         String limitSetting = settingsService.getValueForKey(SettingsServiceBean.Key.GuestbookResponsesPageDisplayLimit);
         return getLongLimitFromStringOrDefault(limitSetting, DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT);
     }
-    
-    public long getUploadLogoSizeLimit(){
+
+    public long getUploadLogoSizeLimit() {
         return 500000;
     }
 
@@ -497,10 +499,10 @@ public long getThumbnailSizeLimitPDF() {
     }
 
     public static long getThumbnailSizeLimit(String type) {
-        String option = null; 
-        
-        //get options via jvm options
-        
+        String option = null;
+
+        // get options via jvm options
+
         if ("Image".equals(type)) {
             option = System.getProperty("dataverse.dataAccess.thumbnail.image.limit");
             return getLongLimitFromStringOrDefault(option, DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE);
@@ -512,19 +514,19 @@ public static long getThumbnailSizeLimit(String type) {
         // Zero (0) means no limit.
         return getLongLimitFromStringOrDefault(option, 0L);
     }
-    
+
     public boolean isThumbnailGenerationDisabledForType(String type) {
         return getThumbnailSizeLimit(type) == -1l;
     }
-    
+
     public boolean isThumbnailGenerationDisabledForImages() {
         return isThumbnailGenerationDisabledForType("Image");
     }
-    
+
     public boolean isThumbnailGenerationDisabledForPDF() {
         return isThumbnailGenerationDisabledForType("PDF");
     }
-    
+
     public String getApplicationTermsOfUse() {
         String language = BundleUtil.getCurrentLocale().getLanguage();
         String saneDefaultForAppTermsOfUse = BundleUtil.getStringFromBundle("system.app.terms");
@@ -532,9 +534,9 @@ public String getApplicationTermsOfUse() {
         // value, or as a better default than the saneDefaultForAppTermsOfUse if there
         // is no language-specific value
         String appTermsOfUse = settingsService.getValueForKey(SettingsServiceBean.Key.ApplicationTermsOfUse, saneDefaultForAppTermsOfUse);
-        //Now get the language-specific value if it exists
+        // Now get the language-specific value if it exists
         if (language != null && !language.equalsIgnoreCase(BundleUtil.getDefaultLocale().getLanguage())) {
-            appTermsOfUse = settingsService.getValueForKey(SettingsServiceBean.Key.ApplicationTermsOfUse, language,	appTermsOfUse);
+            appTermsOfUse = settingsService.getValueForKey(SettingsServiceBean.Key.ApplicationTermsOfUse, language, appTermsOfUse);
         }
         return appTermsOfUse;
     }
@@ -545,7 +547,7 @@ public String getApiTermsOfUse() {
         return apiTermsOfUse;
     }
 
-    // TODO: 
+    // TODO:
     // remove this method!
     // pages should be using settingsWrapper.get(":ApplicationPrivacyPolicyUrl") instead. -- 4.2.1
     public String getApplicationPrivacyPolicyUrl() {
@@ -564,10 +566,10 @@ public boolean isFilesOnDatasetPageFromSolr() {
         return settingsService.isTrueForKey(SettingsServiceBean.Key.FilesOnDatasetPageFromSolr, safeDefaultIfKeyNotFound);
     }
 
-    public Long getMaxFileUploadSizeForStore(String driverId){
-         return settingsService.getValueForCompoundKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes, driverId);
-     }
-    
+    public Long getMaxFileUploadSizeForStore(String driverId) {
+        return settingsService.getValueForCompoundKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes, driverId);
+    }
+
     public Integer getSearchHighlightFragmentSize() {
         String fragSize = settingsService.getValueForKey(SettingsServiceBean.Key.SearchHighlightFragmentSize);
         if (fragSize != null) {
@@ -581,12 +583,12 @@ public Integer getSearchHighlightFragmentSize() {
     }
 
     public long getTabularIngestSizeLimit() {
-        // This method will return the blanket ingestable size limit, if 
-        // set on the system. I.e., the universal limit that applies to all 
-        // tabular ingests, regardless of fromat: 
-        
-        String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); 
-        
+        // This method will return the blanket ingestable size limit, if
+        // set on the system. I.e., the universal limit that applies to all
+        // tabular ingests, regardless of fromat:
+
+        String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
+
         if (limitEntry != null) {
             try {
                 Long sizeOption = new Long(limitEntry);
@@ -595,48 +597,48 @@ public long getTabularIngestSizeLimit() {
                 logger.warning("Invalid value for TabularIngestSizeLimit option? - " + limitEntry);
             }
         }
-        // -1 means no limit is set; 
-        // 0 on the other hand would mean that ingest is fully disabled for 
-        // tabular data. 
-        return -1; 
+        // -1 means no limit is set;
+        // 0 on the other hand would mean that ingest is fully disabled for
+        // tabular data.
+        return -1;
     }
-    
+
     public long getTabularIngestSizeLimit(String formatName) {
         // This method returns the size limit set specifically for this format name,
-        // if available, otherwise - the blanket limit that applies to all tabular 
-        // ingests regardless of a format. 
-        
+        // if available, otherwise - the blanket limit that applies to all tabular
+        // ingests regardless of a format.
+
         if (formatName == null || formatName.equals("")) {
-            return getTabularIngestSizeLimit(); 
+            return getTabularIngestSizeLimit();
         }
-        
+
         String limitEntry = settingsService.get(SettingsServiceBean.Key.TabularIngestSizeLimit.toString() + ":" + formatName); 
-                
+
         if (limitEntry != null) {
             try {
                 Long sizeOption = new Long(limitEntry);
                 return sizeOption;
             } catch (NumberFormatException nfe) {
-                logger.warning("Invalid value for TabularIngestSizeLimit:" + formatName + "? - " + limitEntry );
+                logger.warning("Invalid value for TabularIngestSizeLimit:" + formatName + "? - " + limitEntry);
             }
         }
-        
-        return getTabularIngestSizeLimit();        
+
+        return getTabularIngestSizeLimit();
     }
 
     public boolean isOAIServerEnabled() {
         boolean defaultResponse = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.OAIServerEnabled, defaultResponse);
     }
-    
+
     public void enableOAIServer() {
         settingsService.setValueForKey(SettingsServiceBean.Key.OAIServerEnabled, "true");
     }
-    
+
     public void disableOAIServer() {
         settingsService.deleteValueForKey(SettingsServiceBean.Key.OAIServerEnabled);
-    }   
-    
+    }
+
     public boolean isTimerServer() {
         String optionValue = System.getProperty(JVM_TIMER_SERVER_OPTION);
         if ("true".equalsIgnoreCase(optionValue)) {
@@ -704,11 +706,12 @@ public String getOAuth2CallbackUrl() {
         }
         return saneDefault;
     }
-    
+
     public boolean isShibPassiveLoginEnabled() {
         boolean defaultResponse = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.ShibPassiveLoginEnabled, defaultResponse);
     }
+
     public boolean isShibAttributeCharacterSetConversionEnabled() {
         boolean defaultResponse = true;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.ShibAttributeCharacterSetConversionEnabled, defaultResponse);
@@ -734,7 +737,7 @@ public String getPVDictionaries() {
     public int getPVGoodStrength() {
         // FIXME: Change this to 21 to match Harvard's requirements or implement a way to disable the rule (0 or -1) and have the default be disabled.
         int goodStrengthLength = 20;
-        //String _goodStrengthLength = System.getProperty("pv.goodstrength", settingsService.get(SettingsServiceBean.Key.PVGoodStrength.toString()));
+        // String _goodStrengthLength = System.getProperty("pv.goodstrength", settingsService.get(SettingsServiceBean.Key.PVGoodStrength.toString()));
         String _goodStrengthLength = settingsService.get(SettingsServiceBean.Key.PVGoodStrength.toString());
         try {
             goodStrengthLength = Integer.parseInt(_goodStrengthLength);
@@ -862,9 +865,7 @@ public enum FileUploadMethods {
          * Upload through Globus of large files
          */
 
-        GLOBUS("globus")
-        ;
-
+        GLOBUS("globus");
 
         private final String text;
 
@@ -887,8 +888,7 @@ public static FileUploadMethods fromString(String text) {
         public String toString() {
             return text;
         }
-        
-        
+
     }
 
     /**
@@ -904,8 +904,8 @@ public enum FileDownloadMethods {
          */
         RSYNC("rsal/rsync"),
         NATIVE("native/http"),
-        GLOBUS("globus")
-        ;
+        GLOBUS("globus");
+
         private final String text;
 
         private FileDownloadMethods(final String text) {
@@ -927,27 +927,28 @@ public static FileUploadMethods fromString(String text) {
         public String toString() {
             return text;
         }
-        
+
     }
-    
+
     public enum DataFilePIDFormat {
         DEPENDENT("DEPENDENT"),
         INDEPENDENT("INDEPENDENT");
+
         private final String text;
 
         public String getText() {
             return text;
         }
-        
-        private DataFilePIDFormat(final String text){
+
+        private DataFilePIDFormat(final String text) {
             this.text = text;
         }
-        
+
         @Override
         public String toString() {
             return text;
         }
-        
+
     }
 
     /**
@@ -987,44 +988,44 @@ public String toString() {
 
     }
 
-    public boolean isPublicInstall(){
+    public boolean isPublicInstall() {
         boolean saneDefault = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, saneDefault);
     }
-    
-    public boolean isRsyncUpload(){
+
+    public boolean isRsyncUpload() {
         return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), true);
     }
 
-    public boolean isGlobusUpload(){
+    public boolean isGlobusUpload() {
         return getMethodAvailable(FileUploadMethods.GLOBUS.toString(), true);
     }
 
     // Controls if HTTP upload is enabled for both GUI and API.
-    public boolean isHTTPUpload(){       
+    public boolean isHTTPUpload() {
         return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), true);
     }
-    
-    public boolean isRsyncOnly(){
+
+    public boolean isRsyncOnly() {
         String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods);
-        if(downloadMethods == null){
+        if (downloadMethods == null) {
             return false;
         }
-        if (!downloadMethods.toLowerCase().equals(SystemConfig.FileDownloadMethods.RSYNC.toString())){
+        if (!downloadMethods.toLowerCase().equals(SystemConfig.FileDownloadMethods.RSYNC.toString())) {
             return false;
         }
         String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods);
-        if (uploadMethods==null){
+        if (uploadMethods == null) {
             return false;
         } else {
-           return  Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size() == 1 && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString());
+            return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size() == 1 && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString());
         }
     }
-    
+
     public boolean isRsyncDownload() {
         return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), false);
     }
-    
+
     public boolean isHTTPDownload() {
         return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), false);
     }
@@ -1032,14 +1033,14 @@ public boolean isHTTPDownload() {
     public boolean isGlobusDownload() {
         return getMethodAvailable(FileUploadMethods.GLOBUS.toString(), false);
     }
-    
+
     public boolean isGlobusFileDownload() {
         return (isGlobusDownload() && settingsService.isTrueForKey(SettingsServiceBean.Key.GlobusSingleFileTransfer, false));
     }
 
     public List<String> getGlobusStoresList() {
-    String globusStores = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusStores, "");
-    return Arrays.asList(globusStores.split("\\s*,\\s*"));
+        String globusStores = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusStores, "");
+        return Arrays.asList(globusStores.split("\\s*,\\s*"));
     }
 
     private Boolean getMethodAvailable(String method, boolean upload) {
@@ -1051,31 +1052,32 @@ private Boolean getMethodAvailable(String method, boolean upload) {
             return Arrays.asList(methods.toLowerCase().split("\\s*,\\s*")).contains(method);
         }
     }
-    
-    public Integer getUploadMethodCount(){
-        String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods); 
-        if (uploadMethods==null){
+
+    public Integer getUploadMethodCount() {
+        String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods);
+        if (uploadMethods == null) {
             return 0;
         } else {
-           return  Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size();
-        }       
+            return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size();
+        }
     }
-    public boolean isDataFilePIDSequentialDependent(){
+
+    public boolean isDataFilePIDSequentialDependent() {
         String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString");
         String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
-        if (doiIdentifierType.equals("storedProcGenerated") && doiDataFileFormat.equals("DEPENDENT")){
+        if (doiIdentifierType.equals("storedProcGenerated") && doiDataFileFormat.equals("DEPENDENT")) {
             return true;
         }
         return false;
     }
-    
+
     public int getPIDAsynchRegFileCount() {
         String fileCount = settingsService.getValueForKey(SettingsServiceBean.Key.PIDAsynchRegFileCount, "10");
         int retVal = 10;
         try {
             retVal = Integer.parseInt(fileCount);
-        } catch (NumberFormatException e) {           
-            //if no number in the setting we'll return 10
+        } catch (NumberFormatException e) {
+            // if no number in the setting we'll return 10
         }
         return retVal;
     }
@@ -1089,13 +1091,13 @@ public boolean isFilePIDsEnabled() {
         boolean safeDefaultIfKeyNotFound = true;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.FilePIDsEnabled, safeDefaultIfKeyNotFound);
     }
-    
+
     public boolean isIndependentHandleService() {
         boolean safeDefaultIfKeyNotFound = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.IndependentHandleService, safeDefaultIfKeyNotFound);
-    
+
     }
-    
+
     public String getHandleAuthHandle() {
         String handleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null);
         return handleAuthHandle;
@@ -1105,61 +1107,61 @@ public String getMDCLogPath() {
         String mDCLogPath = settingsService.getValueForKey(SettingsServiceBean.Key.MDCLogPath, null);
         return mDCLogPath;
     }
-    
+
     public boolean isDatafileValidationOnPublishEnabled() {
         boolean safeDefaultIfKeyNotFound = true;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.FileValidationOnPublishEnabled, safeDefaultIfKeyNotFound);
     }
 
-	public boolean directUploadEnabled(DvObjectContainer container) {
+    public boolean directUploadEnabled(DvObjectContainer container) {
         // this method is used in UI only, therfore "dataverse.files." + driverId + ".allow-out-of-band-upload" is not used here
     	return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
-	}
-	
-	public String getDataCiteRestApiUrlString() {
-		//As of 5.0 the 'doi.dataciterestapiurlstring' is the documented jvm option. Prior versions used 'doi.mdcbaseurlstring' or were hardcoded to api.datacite.org, so the defaults are for backward compatibility.
+    }
+
+    public String getDataCiteRestApiUrlString() {
+		// As of 5.0 the 'doi.dataciterestapiurlstring' is the documented jvm option. Prior versions used 'doi.mdcbaseurlstring' or were hardcoded to api.datacite.org, so the defaults are for backward compatibility.
         return System.getProperty("doi.dataciterestapiurlstring", System.getProperty("doi.mdcbaseurlstring", "https://api.datacite.org"));
-	}
-        
+    }
+
     public boolean isExternalDataverseValidationEnabled() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataValidatorScript) != null;
-        // alternatively, we can also check if the script specified exists, 
+        // alternatively, we can also check if the script specified exists,
         // and is executable. -- ?
     }
-    
+
     public boolean isExternalDatasetValidationEnabled() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DatasetMetadataValidatorScript) != null;
-        // alternatively, we can also check if the script specified exists, 
+        // alternatively, we can also check if the script specified exists,
         // and is executable. -- ?
     }
-    
+
     public String getDataverseValidationExecutable() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataValidatorScript);
     }
-    
+
     public String getDatasetValidationExecutable() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DatasetMetadataValidatorScript);
     }
-    
+
     public String getDataverseValidationFailureMsg() {
         String defaultMessage = "This dataverse collection cannot be published because it has failed an external metadata validation test.";
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataPublishValidationFailureMsg, defaultMessage);
     }
-    
+
     public String getDataverseUpdateValidationFailureMsg() {
         String defaultMessage = "This dataverse collection cannot be updated because it has failed an external metadata validation test.";
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataUpdateValidationFailureMsg, defaultMessage);
     }
-    
+
     public String getDatasetValidationFailureMsg() {
         String defaultMessage = "This dataset cannot be published because it has failed an external metadata validation test.";
         return settingsService.getValueForKey(SettingsServiceBean.Key.DatasetMetadataValidationFailureMsg, defaultMessage);
     }
-    
+
     public boolean isExternalValidationAdminOverrideEnabled() {
         return "true".equalsIgnoreCase(settingsService.getValueForKey(SettingsServiceBean.Key.ExternalValidationAdminOverride));
     }
-    
+
     public long getDatasetValidationSizeLimit() {
         String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.DatasetChecksumValidationSizeLimit);
 
@@ -1189,6 +1191,7 @@ public long getFileValidationSizeLimit() {
         // -1 means no limit is set;
         return -1;
     }
+
     public Map<String, String[]> getCurationLabels() {
         Map<String, String[]> labelMap = new HashMap<String, String[]>();
         String setting = settingsService.getValueForKey(SettingsServiceBean.Key.AllowedCurationLabels, "");
@@ -1229,15 +1232,15 @@ public Map<String, String[]> getCurationLabels() {
         }
         return labelMap;
     }
-    
+
     public boolean isSignupDisabledForRemoteAuthProvider(String providerId) {
-        Boolean ret =  settingsService.getValueForCompoundKeyAsBoolean(SettingsServiceBean.Key.AllowRemoteAuthSignUp, providerId);
-        
-        // we default to false - i.e., "not disabled" if the setting is not present: 
+        Boolean ret = settingsService.getValueForCompoundKeyAsBoolean(SettingsServiceBean.Key.AllowRemoteAuthSignUp, providerId);
+
+        // we default to false - i.e., "not disabled" if the setting is not present:
         if (ret == null) {
-            return false; 
+            return false;
         }
-        
-        return !ret; 
+
+        return !ret;
     }
 }

From 8578de173b63dbde3bb5440147422783621fbee9 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 10:10:57 +0100
Subject: [PATCH 009/922] tab character removed

---
 src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index f3d8e46b004..1edf5a0fb6e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -1105,7 +1105,7 @@ public String getHandleAuthHandle() {
 
     public String getMDCLogPath() {
         String mDCLogPath = settingsService.getValueForKey(SettingsServiceBean.Key.MDCLogPath, null);
-        return mDCLogPath;
+        return mDCLogPath;this method is used
     }
 
     public boolean isDatafileValidationOnPublishEnabled() {
@@ -1115,7 +1115,7 @@ public boolean isDatafileValidationOnPublishEnabled() {
 
     public boolean directUploadEnabled(DvObjectContainer container) {
         // this method is used in UI only, therfore "dataverse.files." + driverId + ".allow-out-of-band-upload" is not used here
-    	return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
+        return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
     }
 
     public String getDataCiteRestApiUrlString() {

From f2e75db13bcff1f5a5bc7d5cfc958db04be745c0 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 10:15:03 +0100
Subject: [PATCH 010/922] tab character removed

---
 src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 1edf5a0fb6e..0ab99c0de6a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -1105,7 +1105,7 @@ public String getHandleAuthHandle() {
 
     public String getMDCLogPath() {
         String mDCLogPath = settingsService.getValueForKey(SettingsServiceBean.Key.MDCLogPath, null);
-        return mDCLogPath;this method is used
+        return mDCLogPath;
     }
 
     public boolean isDatafileValidationOnPublishEnabled() {

From bff889d3864ca10f7dc4f7ae84595e40a2b70d34 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 10:25:39 +0100
Subject: [PATCH 011/922] tab character removed

---
 src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 0ab99c0de6a..7d7006e708e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -1119,7 +1119,7 @@ public boolean directUploadEnabled(DvObjectContainer container) {
     }
 
     public String getDataCiteRestApiUrlString() {
-		// As of 5.0 the 'doi.dataciterestapiurlstring' is the documented jvm option. Prior versions used 'doi.mdcbaseurlstring' or were hardcoded to api.datacite.org, so the defaults are for backward compatibility.
+        // As of 5.0 the 'doi.dataciterestapiurlstring' is the documented jvm option. Prior versions used 'doi.mdcbaseurlstring' or were hardcoded to api.datacite.org, so the defaults are for backward compatibility.
         return System.getProperty("doi.dataciterestapiurlstring", System.getProperty("doi.mdcbaseurlstring", "https://api.datacite.org"));
     }
 

From ad4bb5107fcb14b8c4ebb7f7fd57186511577548 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 11:51:31 +0100
Subject: [PATCH 012/922] renamed jvm option: allow-out-of-band-upload ->
 upload-out-of-band

---
 doc/sphinx-guides/source/installation/config.rst         | 9 +++++----
 .../edu/harvard/iq/dataverse/dataaccess/StorageIO.java   | 2 +-
 .../java/edu/harvard/iq/dataverse/util/FileUtil.java     | 2 +-
 .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 4f15ad81190..62cc984bc56 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -249,11 +249,11 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
-When using integration tools, dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.allow-out-of-band-upload`` JVM option to ``true``.
+When using integration tools, dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
 Files can be then uploaded by an integration tool with ``datasets/{id}/add`` api call, or uploaded directly to the storage and registerd in a dataset afterwards using the ``datasets/{id}/addFiles`` api call.
-Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``allow-out-of-band-upload`` and will enable direct upload even with ``allow-out-of-band-upload`` not set (or set to false).
-In other words, ``dataverse.files.\<id\>.allow-out-of-band-upload`` option opens the ``datasets/{id}/add`` and ``datasets/{id}/addFiles`` api endpoints without redirecting uploads in the UI.
-Enabling the ``upload-redirect`` option allows then direct upload automatically, without the need of enabling the ``allow-out-of-band-upload`` (setting it to ``false`` does not have any effect in that case).
+Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``upload-out-of-band`` and will enable direct upload even with ``upload-out-of-band`` not set (or set to false).
+In other words, ``dataverse.files.\<id\>.upload-out-of-band`` option opens the ``datasets/{id}/add`` and ``datasets/{id}/addFiles`` api endpoints without redirecting uploads in the UI.
+Enabling the ``upload-redirect`` option allows then direct upload automatically, without the need of enabling the ``upload-out-of-band`` (setting it to ``false`` does not have any effect in that case).
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 
@@ -552,6 +552,7 @@ List of S3 Storage Options
     dataverse.files.<id>.bucket-name             <?>                 The bucket name. See above.                                                 (none)
     dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download or proxy through Dataverse.                          ``false``
     dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset in the S3 store.           ``false``
+    dataverse.files.<id>.upload-out-of-band      ``true``/``false``  Enable direct upload of files added to a dataset via API only.              ``false``
     dataverse.files.<id>.ingestsizelimit         <size in bytes>     Maximum size of directupload files that should be ingested                  (none)
     dataverse.files.<id>.url-expiration-minutes  <?>                 If direct uploads/downloads: time until links expire. Optional.             60
     dataverse.files.<id>.min-part-size           <?>                 Multipart direct uploads will occur for files larger than this. Optional.   ``1024**3``
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index a2ff546ef0a..85ca97d5f15 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -606,7 +606,7 @@ public static String getDriverPrefix(String driverId) {
     
     public static boolean isDirectUploadEnabled(String driverId) {
         return Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".allow-out-of-band-upload"));
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
     }
     
     //Check that storageIdentifier is consistent with store's config
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index db82df72b8a..9b549901d55 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1749,7 +1749,7 @@ public static boolean isPackageFile(DataFile dataFile) {
     public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	String driverId = dataset.getEffectiveStorageDriverId();
     	boolean directEnabled = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".allow-out-of-band-upload"));
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".aupload-out-of-band"));
     	//Should only be requested when it is allowed, but we'll log a warning otherwise
     	if(!directEnabled) {
     		logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId());
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 7d7006e708e..b45ad50ab1d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -1114,7 +1114,7 @@ public boolean isDatafileValidationOnPublishEnabled() {
     }
 
     public boolean directUploadEnabled(DvObjectContainer container) {
-        // this method is used in UI only, therfore "dataverse.files." + driverId + ".allow-out-of-band-upload" is not used here
+        // this method is used in UI only, therfore "dataverse.files." + driverId + ".upload-out-of-band" is not used here
         return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
     }
 

From 49102ada3380863d115f5167343eb97446b35872 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 12:49:36 +0100
Subject: [PATCH 013/922] linking to api documentation

---
 doc/sphinx-guides/source/api/native-api.rst      | 1 +
 doc/sphinx-guides/source/installation/config.rst | 5 ++---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 6d68d648cb3..0341b6e07d1 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2348,6 +2348,7 @@ The fully expanded example above (without environment variables) looks like this
 Note: The ``id`` returned in the json response is the id of the file metadata version.
 
 
+.. _add-file-metadata-api:
 
 Adding File Metadata
 ~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 62cc984bc56..b074a180c8f 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -250,10 +250,9 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
 When using integration tools, dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
-Files can be then uploaded by an integration tool with ``datasets/{id}/add`` api call, or uploaded directly to the storage and registerd in a dataset afterwards using the ``datasets/{id}/addFiles`` api call.
+Files can be then uploaded by an integration tool with :ref:`add-file-api` api call, or uploaded directly to the storage and registerd in a dataset afterwards using the :ref:`add-file-metadata-api` api call.
 Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``upload-out-of-band`` and will enable direct upload even with ``upload-out-of-band`` not set (or set to false).
-In other words, ``dataverse.files.\<id\>.upload-out-of-band`` option opens the ``datasets/{id}/add`` and ``datasets/{id}/addFiles`` api endpoints without redirecting uploads in the UI.
-Enabling the ``upload-redirect`` option allows then direct upload automatically, without the need of enabling the ``upload-out-of-band`` (setting it to ``false`` does not have any effect in that case).
+In other words, ``dataverse.files.\<id\>.upload-out-of-band`` option opens the :ref:`add-file-api` and :ref:`add-file-metadata-api` api endpoints without redirecting uploads in the UI.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 

From e9d6df0bb6f23f4f4a8e7fe53213c91596980332 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 14:11:19 +0100
Subject: [PATCH 014/922] some improvements in the documentation

---
 doc/sphinx-guides/source/api/native-api.rst      | 4 ++++
 doc/sphinx-guides/source/installation/config.rst | 8 ++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 0341b6e07d1..f075acf40f6 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1481,6 +1481,8 @@ In practice, you only need one the ``dataset_id`` or the ``persistentId``. The e
     print r.json()
     print r.status_code
 
+This API call might result in an error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. It can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
+
 .. _add-remote-file-api:
     
 Add a Remote File to a Dataset
@@ -2391,6 +2393,8 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/:persistentId/addFiles?persistentId=doi:10.5072/FK2/7U7YBV -F jsonData='[{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}, {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]'
 
+This API call might result in an error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. It can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
+
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index b074a180c8f..d3a22453453 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -249,10 +249,10 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
-When using integration tools, dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
-Files can be then uploaded by an integration tool with :ref:`add-file-api` api call, or uploaded directly to the storage and registerd in a dataset afterwards using the :ref:`add-file-metadata-api` api call.
-Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``upload-out-of-band`` and will enable direct upload even with ``upload-out-of-band`` not set (or set to false).
-In other words, ``dataverse.files.\<id\>.upload-out-of-band`` option opens the :ref:`add-file-api` and :ref:`add-file-metadata-api` api endpoints without redirecting uploads in the UI.
+A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
+This option allows adding files with the :ref:`add-file-api` call. It also allows registering the metadata of a file with the :ref:`add-file-metadata-api` call for a file uploaded directly to the storage.
+Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``upload-out-of-band`` option and will enable direct upload even with ``upload-out-of-band`` option not set (or set to false).
+When neither of the two option is enabled, adding files with API will not be possible and will result with the "Dataset store configuration does not allow provided storageIdentifier" error.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 

From dc64aa23c3d4c364f46ad6e695e38ed3311455eb Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 17 Nov 2022 17:47:13 +0100
Subject: [PATCH 015/922] documentation improvements by Dieuwertje

---
 doc/sphinx-guides/source/api/native-api.rst      | 4 ++--
 doc/sphinx-guides/source/installation/config.rst | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index f075acf40f6..54e47a29b9d 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1481,7 +1481,7 @@ In practice, you only need one the ``dataset_id`` or the ``persistentId``. The e
     print r.json()
     print r.status_code
 
-This API call might result in an error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. It can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
+This API call might result in the following error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. This error can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
 
 .. _add-remote-file-api:
     
@@ -2393,7 +2393,7 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/:persistentId/addFiles?persistentId=doi:10.5072/FK2/7U7YBV -F jsonData='[{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}, {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]'
 
-This API call might result in an error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. It can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
+This API call might result in the following error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. This error can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
 
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index d3a22453453..4eadcc8ed9d 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -250,9 +250,9 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
-This option allows adding files with the :ref:`add-file-api` call. It also allows registering the metadata of a file with the :ref:`add-file-metadata-api` call for a file uploaded directly to the storage.
-Notice that using S3-storage with ``dataverse.files.\<id\>.upload-redirect`` JVM option enabled supersedes the ``upload-out-of-band`` option and will enable direct upload even with ``upload-out-of-band`` option not set (or set to false).
-When neither of the two option is enabled, adding files with API will not be possible and will result with the "Dataset store configuration does not allow provided storageIdentifier" error.
+This option allows API users to add files with the :ref:`add-file-api` call. It also allows API users to register the metadata of a file with the :ref:`add-file-metadata-api` call for a file that was uploaded directly to the storage.
+Note that if a Dataverse installation uses S3-storage while the ``dataverse.files.\<id\>.upload-redirect`` JVM option is enabled, the ``upload-out-of-band`` setting is overruled. This results in direct upload being enabled even with the ``upload-out-of-band`` option not set (or set to false).
+When the ``upload-out-of-band`` option is not set to ``true`` and it isn't being overruled by the previously mentioned combination, adding files using the API will not be possible and will return the "Dataset store configuration does not allow provided storageIdentifier" error.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 

From 085fb8f44503d69354b5cb8f5793d8144dbde0e1 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Mon, 21 Nov 2022 09:53:01 +0100
Subject: [PATCH 016/922] improvements in the documentation

---
 doc/sphinx-guides/source/installation/config.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 4eadcc8ed9d..467872bfdd4 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -251,6 +251,9 @@ A Dataverse installation may also be configured to reference some files (e.g. la
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
 This option allows API users to add files with the :ref:`add-file-api` call. It also allows API users to register the metadata of a file with the :ref:`add-file-metadata-api` call for a file that was uploaded directly to the storage.
+
+The option is useful in cases in which an S3 storage is not used or made public, as required by the ``dataverse.files.\<id\>.upload-redirect`` option. An example would be building a tool for synchronizing datasets with files from a third-party repository. In such a case, the tool would upload files directly to the storage, and then use :ref:`add-file-metadata-api` to link them to a dataset.
+
 Note that if a Dataverse installation uses S3-storage while the ``dataverse.files.\<id\>.upload-redirect`` JVM option is enabled, the ``upload-out-of-band`` setting is overruled. This results in direct upload being enabled even with the ``upload-out-of-band`` option not set (or set to false).
 When the ``upload-out-of-band`` option is not set to ``true`` and it isn't being overruled by the previously mentioned combination, adding files using the API will not be possible and will return the "Dataset store configuration does not allow provided storageIdentifier" error.
 

From d870e202dccac268cc3f099277559d3e473b7944 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 9 Jan 2023 13:18:06 +0100
Subject: [PATCH 017/922] chore(deps): upgrade Nimbus OIDC SDK to latest 10.4
 release #9268

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8b6f98c5896..a5d52fd7545 100644
--- a/pom.xml
+++ b/pom.xml
@@ -381,7 +381,7 @@
         <dependency>
             <groupId>com.nimbusds</groupId>
             <artifactId>oauth2-oidc-sdk</artifactId>
-            <version>9.41.1</version>
+            <version>10.4</version>
         </dependency>
         <!-- New and Improved GDCC XOAI library! --> 
         <dependency>

From 2ee66618ed77d55878300a7baaa4fa4a94ac7162 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 9 Jan 2023 15:52:14 +0100
Subject: [PATCH 018/922] style(oidc): make class fields final in
 OIDCAuthProvider

These values should not be changed once the provider has been initialized.
---
 .../oauth2/oidc/OIDCAuthProvider.java         | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
index a9c44010950..4b6c575cfaf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
@@ -54,15 +54,15 @@ public class OIDCAuthProvider extends AbstractOAuth2AuthenticationProvider {
     protected String title = "Open ID Connect";
     protected List<String> scope = Arrays.asList("openid", "email", "profile");
     
-    Issuer issuer;
-    ClientAuthentication clientAuth;
-    OIDCProviderMetadata idpMetadata;
+    final Issuer issuer;
+    final ClientAuthentication clientAuth;
+    final OIDCProviderMetadata idpMetadata;
     
     public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEndpointURL) throws AuthorizationSetupException {
         this.clientSecret = aClientSecret; // nedded for state creation
         this.clientAuth = new ClientSecretBasic(new ClientID(aClientId), new Secret(aClientSecret));
         this.issuer = new Issuer(issuerEndpointURL);
-        getMetadata();
+        this.idpMetadata = getMetadata();
     }
     
     /**
@@ -74,7 +74,9 @@ public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEnd
      * @return false
      */
     @Override
-    public boolean isDisplayIdentifier() { return false; }
+    public boolean isDisplayIdentifier() {
+        return false;
+    }
     
     /**
      * Setup metadata from OIDC provider during creation of the provider representation
@@ -82,9 +84,14 @@ public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEnd
      * @throws IOException when sth. goes wrong with the retrieval
      * @throws ParseException when the metadata is not parsable
      */
-    void getMetadata() throws AuthorizationSetupException {
+    OIDCProviderMetadata getMetadata() throws AuthorizationSetupException {
         try {
-            this.idpMetadata = getMetadata(this.issuer);
+            var metadata = getMetadata(this.issuer);
+            // Assert that the provider supports the code flow
+            if (metadata.getResponseTypes().stream().noneMatch(ResponseType::impliesCodeFlow)) {
+                throw new AuthorizationSetupException("OIDC provider at "+this.issuer.getValue()+" does not support code flow, disabling.");
+            }
+            return metadata;
         } catch (IOException ex) {
             logger.severe("OIDC provider metadata at \"+issuerEndpointURL+\" not retrievable: "+ex.getMessage());
             throw new AuthorizationSetupException("OIDC provider metadata at "+this.issuer.getValue()+" not retrievable.");
@@ -92,11 +99,6 @@ void getMetadata() throws AuthorizationSetupException {
             logger.severe("OIDC provider metadata at \"+issuerEndpointURL+\" not parsable: "+ex.getMessage());
             throw new AuthorizationSetupException("OIDC provider metadata at "+this.issuer.getValue()+" not parsable.");
         }
-    
-        // Assert that the provider supports the code flow
-        if (! this.idpMetadata.getResponseTypes().stream().filter(idp -> idp.impliesCodeFlow()).findAny().isPresent()) {
-            throw new AuthorizationSetupException("OIDC provider at "+this.issuer.getValue()+" does not support code flow, disabling.");
-        }
     }
     
     /**

From 0c7db6614669ecc40e96ffb029be4f21ed04f4db Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:27:55 +0100
Subject: [PATCH 019/922] chore(deps): update Testcontainers to latest version

Also updating Postgres Server version in "tc" Maven profile.
---
 modules/dataverse-parent/pom.xml | 2 +-
 pom.xml                          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 3911e9d5bbb..e316a5508ce 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -167,7 +167,7 @@
         <gdcc.xoai.version>5.0.0-RC2</gdcc.xoai.version>
     
         <!-- Testing dependencies -->
-        <testcontainers.version>1.15.0</testcontainers.version>
+        <testcontainers.version>1.17.6</testcontainers.version>
         <smallrye-mpconfig.version>2.10.1</smallrye-mpconfig.version>
     
         <junit.version>4.13.1</junit.version>
diff --git a/pom.xml b/pom.xml
index a5d52fd7545..56871c7fd56 100644
--- a/pom.xml
+++ b/pom.xml
@@ -757,7 +757,7 @@
             <id>tc</id>
             <properties>
                 <skipUnitTests>true</skipUnitTests>
-                <postgresql.server.version>9.6</postgresql.server.version>
+                <postgresql.server.version>13.0</postgresql.server.version>
             </properties>
             <build>
                 <plugins>

From 5681d24520ac017eb925bc058ecaef877eedd14b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:28:52 +0100
Subject: [PATCH 020/922] chore(deps): add Keycloak Testcontainer module for
 tests

---
 pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/pom.xml b/pom.xml
index 56871c7fd56..63c362ba904 100644
--- a/pom.xml
+++ b/pom.xml
@@ -570,6 +570,12 @@
             <artifactId>postgresql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.github.dasniko</groupId>
+            <artifactId>testcontainers-keycloak</artifactId>
+            <version>2.4.0</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>

From 9f534c4e4a59d7b33b9a0e4a5a876819e9278c47 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:32:12 +0100
Subject: [PATCH 021/922] feat(tests): extend JvmSetting helper for test class
 method references

Instead of only allowing to supply static String values for a setting,
also allow referencing a static method in the test class to retrieve dynamic data.

This is inspired by the JUnit5 MethodSource example.
---
 .../iq/dataverse/util/testing/JvmSetting.java |  6 ++++-
 .../util/testing/JvmSettingExtension.java     | 25 ++++++++++++++++++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSetting.java b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSetting.java
index f54cadaf253..85b10489f15 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSetting.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSetting.java
@@ -39,6 +39,8 @@
 @ResourceLock(value = Resources.SYSTEM_PROPERTIES, mode = ResourceAccessMode.READ_WRITE)
 public @interface JvmSetting {
     
+    static final String PLACEHOLDER = "NULL";
+    
     /**
      * The key of the system property to be set.
      */
@@ -47,10 +49,12 @@
     /**
      * The value of the system property to be set.
      */
-    String value();
+    String value() default PLACEHOLDER;
     
     String[] varArgs() default {};
     
+    String method() default PLACEHOLDER;
+    
     /**
      * Containing annotation of repeatable {@code @SetSystemProperty}.
      */
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java
index 56e87589139..17728e75ffc 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java
@@ -5,6 +5,11 @@
 import org.junit.jupiter.api.extension.BeforeTestExecutionCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
 
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import static edu.harvard.iq.dataverse.util.testing.JvmSetting.PLACEHOLDER;
+
 public class JvmSettingExtension implements BeforeTestExecutionCallback, AfterTestExecutionCallback {
     
     private ExtensionContext.Store getStore(ExtensionContext context) {
@@ -28,7 +33,25 @@ public void beforeTestExecution(ExtensionContext extensionContext) throws Except
                 }
                 
                 // set to new value
-                System.setProperty(settingName, setting.value());
+                if (setting.value().equals(PLACEHOLDER) && setting.method().equals(PLACEHOLDER)) {
+                    throw new IllegalArgumentException("You must either provide a value or a method reference " +
+                        "for key JvmSettings." + setting.key());
+                }
+                
+                // retrieve value from static test class method if no setting given
+                if (setting.value().equals(PLACEHOLDER)) {
+                    extensionContext.getTestClass().ifPresent(klass -> {
+                        try {
+                            Method valueMethod = klass.getDeclaredMethod(setting.method());
+                            valueMethod.setAccessible(true);
+                            System.setProperty(settingName, (String)valueMethod.invoke(null));
+                        } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                } else {
+                    System.setProperty(settingName, setting.value());
+                }
             }
         });
     }

From 5cd9f2eb8bd01b88cde28e41c8b27c52656c62b9 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:40:13 +0100
Subject: [PATCH 022/922] doc(dev): add description for method references in
 @JvmSetting helper

---
 doc/sphinx-guides/source/developers/testing.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 4b3d5fd0a55..2d1948449a9 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -89,8 +89,12 @@ For unit tests, the most interesting part is to set a JVM setting just for the c
 Please use the ``@JvmSetting(key = JvmSettings.XXX, value = "")`` annotation on a test method or
 a test class to set and clear the property automatically.
 
-To set arbitrary system properties for the current test, a similar extension
-``@SystemProperty(key = "", value = "")`` has been added.
+Inspired by JUnit's ``@MethodSource`` annotation, you may use ``@JvmSetting(key = JvmSettings.XXX, method = "zzz")``
+to reference a method located in the same test class by name (i. e. ``private static String zzz() {}``) to allow
+retrieving dynamic data instead of String constants only. (Note the requirement for a *static* method!)
+
+To set arbitrary system properties for the current test, a similar extension ``@SystemProperty(key = "", value = "")``
+has been added. (Note: it does not support method references.)
 
 Both extensions will ensure the global state of system properties is non-interfering for
 test executions. Tests using these extensions will be executed in serial.

From ebd8eede980fa1b3cce3e2f30538c9a79c180eb2 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:45:04 +0100
Subject: [PATCH 023/922] feat(settings): add authentication settings for OIDC
 to JvmSettings #9268

---
 .../harvard/iq/dataverse/settings/JvmSettings.java    | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index bc5a73cd958..46b79b06466 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -64,6 +64,17 @@ public enum JvmSettings {
     SCOPE_API(PREFIX, "api"),
     API_SIGNING_SECRET(SCOPE_API, "signing-secret"),
     
+    // AUTH SETTINGS
+    SCOPE_AUTH(PREFIX, "auth"),
+    // AUTH: OIDC SETTINGS
+    SCOPE_OIDC(SCOPE_AUTH, "oidc"),
+    OIDC_ENABLED(SCOPE_OIDC, "enabled"),
+    OIDC_TITLE(SCOPE_OIDC, "title"),
+    OIDC_SUBTITLE(SCOPE_OIDC, "subtitle"),
+    OIDC_AUTH_SERVER_URL(SCOPE_OIDC, "auth-server-url"),
+    OIDC_CLIENT_ID(SCOPE_OIDC, "client-id"),
+    OIDC_CLIENT_SECRET(SCOPE_OIDC, "client-secret"),
+    
     ;
     
     private static final String SCOPE_SEPARATOR = ".";

From 1bff1be736a1362dd779be66415919961fb44599 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:48:18 +0100
Subject: [PATCH 024/922] feat(auth): add OIDC provider provisioning via
 MPCONFIG #9268

Only one provider can be configured via MPCONFIG for now.
The provider is configured with an appropriate ID to distinguish
it from other providers configured via the API.
It can be configured in addition to other OIDC providers when desired.
---
 ...ationProvidersRegistrationServiceBean.java | 10 ++++++++++
 .../OIDCAuthenticationProviderFactory.java    | 20 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java
index 6289865baf0..79dabe1d390 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationProvidersRegistrationServiceBean.java
@@ -17,6 +17,7 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2AuthenticationProviderFactory;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactory;
 import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProviderFactory;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean;
 import java.util.HashMap;
 import java.util.Map;
@@ -121,6 +122,15 @@ public void startup() {
                         logger.log(Level.SEVERE, "Exception setting up the authentication provider '" + row.getId() + "': " + ex.getMessage(), ex);
                     }
         });
+        
+        // Add providers registered via MPCONFIG
+        if (JvmSettings.OIDC_ENABLED.lookupOptional(Boolean.class).orElse(false)) {
+            try {
+                registerProvider(OIDCAuthenticationProviderFactory.buildFromSettings());
+            } catch (AuthorizationSetupException e) {
+                logger.log(Level.SEVERE, "Exception setting up an OIDC auth provider via MicroProfile Config", e);
+            }
+        }
     }
 
     private void registerProviderFactory(AuthenticationProviderFactory aFactory) 
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
index c6d1a28e19d..f4d631adea3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
@@ -5,6 +5,7 @@
 import edu.harvard.iq.dataverse.authorization.providers.AuthenticationProviderFactory;
 import edu.harvard.iq.dataverse.authorization.providers.AuthenticationProviderRow;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2AuthenticationProviderFactory;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 
 import java.util.Map;
 
@@ -44,4 +45,23 @@ public AuthenticationProvider buildProvider( AuthenticationProviderRow aRow ) th
         
         return oidc;
     }
+    
+    /**
+     * Build an OIDC provider from MicroProfile Config provisioned details
+     * @return The configured auth provider
+     * @throws AuthorizationSetupException
+     */
+    public static AuthenticationProvider buildFromSettings() throws AuthorizationSetupException {
+        OIDCAuthProvider oidc = new OIDCAuthProvider(
+            JvmSettings.OIDC_CLIENT_ID.lookup(),
+            JvmSettings.OIDC_CLIENT_SECRET.lookup(),
+            JvmSettings.OIDC_AUTH_SERVER_URL.lookup()
+        );
+        
+        oidc.setId("oidc-mpconfig");
+        oidc.setTitle(JvmSettings.OIDC_TITLE.lookupOptional().orElse("OpenID Connect"));
+        oidc.setSubTitle(JvmSettings.OIDC_SUBTITLE.lookupOptional().orElse("OpenID Connect"));
+        
+        return oidc;
+    }
 }

From fb11096562269d3704dd74504b6e665a6a6a843e Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:48:48 +0100
Subject: [PATCH 025/922] style(auth): slight reformat of OIDC provider factory
 #9268

---
 .../oauth2/oidc/OIDCAuthenticationProviderFactory.java     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
index f4d631adea3..89cf1cb986d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
@@ -38,7 +38,12 @@ public String getInfo() {
     public AuthenticationProvider buildProvider( AuthenticationProviderRow aRow ) throws AuthorizationSetupException {
         Map<String, String> factoryData = OAuth2AuthenticationProviderFactory.parseFactoryData(aRow.getFactoryData());
         
-        OIDCAuthProvider oidc = new OIDCAuthProvider(factoryData.get("clientId"), factoryData.get("clientSecret"), factoryData.get("issuer"));
+        OIDCAuthProvider oidc = new OIDCAuthProvider(
+            factoryData.get("clientId"),
+            factoryData.get("clientSecret"),
+            factoryData.get("issuer")
+        );
+        
         oidc.setId(aRow.getId());
         oidc.setTitle(aRow.getTitle());
         oidc.setSubTitle(aRow.getSubtitle());

From 1fb0f588262a92010c5f0afa52d336a707358a6b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:51:24 +0100
Subject: [PATCH 026/922] test(auth): add integration test for OIDC
 provisioning via MPCONFIG #9268

Using Testcontainers to start a Keycloak instance with our default development realm,
the provider is created using MPCONFIG settings.
---
 .../OIDCAuthenticationProviderFactoryIT.java  | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java

diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
new file mode 100644
index 00000000000..53cfcca2742
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -0,0 +1,37 @@
+package edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc;
+
+import dasniko.testcontainers.keycloak.KeycloakContainer;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@Tag("testcontainers")
+@Testcontainers
+class OIDCAuthenticationProviderFactoryIT {
+    
+    static final String clientId = "oidc-client";
+    static final String clientSecret = "ss6gE8mODCDfqesQaSG3gwUwZqZt547E";
+    static final String realm = "oidc-realm";
+    
+    @Container
+    static KeycloakContainer keycloakContainer = new KeycloakContainer().withRealmImportFile("keycloak/oidc-realm.json");
+    
+    // simple method to retrieve the issuer URL, referenced to by @JvmSetting annotations
+    private static String getAuthUrl() {
+        return keycloakContainer.getAuthServerUrl() + "realms/" + realm;
+    }
+    
+    @Test
+    @JvmSetting(key = JvmSettings.OIDC_CLIENT_ID, value = clientId)
+    @JvmSetting(key = JvmSettings.OIDC_CLIENT_SECRET, value = clientSecret)
+    @JvmSetting(key = JvmSettings.OIDC_AUTH_SERVER_URL, method = "getAuthUrl")
+    void testCreateProvider() throws Exception {
+        OIDCAuthProvider oidcAuthProvider = (OIDCAuthProvider) OIDCAuthenticationProviderFactory.buildFromSettings();
+        assertTrue(oidcAuthProvider.getMetadata().getTokenEndpointURI().toString().startsWith(keycloakContainer.getAuthServerUrl()));
+    }
+}
\ No newline at end of file

From e31dba3da3dc267e963c537da4d0076ed11eee44 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 20:54:39 +0100
Subject: [PATCH 027/922] build(auth): make resources in /conf avail to tests
 #9268

To use data in /conf for tests, adding the folder in Maven to copy them to the test classpath as resources helps to use them in tests very easily.
All dirs under /conf will be copied to the /target/test-classes directory recursively. This also works when running tests in IDEs like IntelliJ.
---
 pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pom.xml b/pom.xml
index 63c362ba904..a26071d253b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -596,6 +596,11 @@
         </dependency>
     </dependencies>
     <build>
+        <testResources>
+            <testResource>
+                <directory>${project.basedir}/conf</directory>
+            </testResource>
+        </testResources>
         <!--        <testResources>
             <testResource>
                 <directory>${project.basedir}/src/main/resources</directory>

From f9861dce9ba584dc1060d2057490f8a65d6a0bc5 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 21:21:16 +0100
Subject: [PATCH 028/922] doc(auth): describe provisioning of OIDC provider via
 MPCONFIG #9268

---
 .../source/installation/oidc.rst              | 64 ++++++++++++++++---
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index a40ef758dc7..c5e6d0f298f 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -16,7 +16,7 @@ Being a standard, you can easily enable the use of any OpenID connect compliant
 Some prominent provider examples:
 
 - `Google <https://developers.google.com/identity/protocols/OpenIDConnect>`_
-- `Microsoft Azure AD <https://docs.microsoft.com/de-de/azure/active-directory/develop/v2-protocols-oidc>`_
+- `Microsoft Azure AD <https://learn.microsoft.com/en-us/azure/active-directory/develop/v2-protocols-oidc>`_
 - `Yahoo <https://developer.yahoo.com/oauth2/guide/openid_connect>`_
 - ORCID `announced support <https://orcid.org/blog/2019/04/17/orcid-openid-connect-and-implicit-authentication>`_
 
@@ -59,7 +59,20 @@ Finding the issuer URL is best done by searching for terms like "discovery" in t
 The discovery document is always located at ``<issuer url>/.well-known/openid-configuration`` (standardized).
 To be sure, you can always lookup the ``issuer`` value inside the live JSON-based discovery document.
 
-Please create a my-oidc-provider.json file like this, replacing every ``<...>`` with your values:
+Note if you work with Keycloak, make sure the base URL is in the following format: ``https://host:port/realms/{realm}``
+where ``{realm}`` has to be replaced by the name of the Keycloak realm.
+
+After adding a provider, the Log In page will by default show the "builtin" provider, but you can adjust this via the
+``:DefaultAuthProvider`` configuration option. For details, see :doc:`config`.
+
+.. hint::
+   In contrast to our :doc:`oauth2`, you can use multiple providers by creating distinct configurations enabled by
+   the same technology and without modifying the Dataverse Software code base (standards for the win!).
+
+Provision via REST API
+^^^^^^^^^^^^^^^^^^^^^^
+
+Please create a ``my-oidc-provider.json`` file like this, replacing every ``<...>`` with your values:
 
 .. code-block:: json
 
@@ -80,10 +93,45 @@ The Dataverse installation will automatically try to load the provider and retri
 You should see the new provider under "Other options" on the Log In page, as described in the :doc:`/user/account`
 section of the User Guide.
 
-By default, the Log In page will show the "builtin" provider, but you can adjust this via the ``:DefaultAuthProvider``
-configuration option. For details, see :doc:`config`.
-
-.. hint::
-   In contrast to our :doc:`oauth2`, you can use multiple providers by creating distinct configurations enabled by
-   the same technology and without modifying the Dataverse Software code base (standards for the win!).
+Provision via MPCONFIG
+^^^^^^^^^^^^^^^^^^^^^^
+
+In case you only require a single OIDC provider, you can also provision an OIDC based provider using MicroProfile Config
+sources. Note that this provider will only be deployed at startup time and (currently) cannot be reconfigured without
+a restart.
+
+The following options are available:
+
+.. list-table::
+  :widths: 25 55 10 10
+  :header-rows: 1
+
+  * - Option
+    - Description
+    - Mandatory
+    - Default
+  * - ``dataverse.auth.oidc.enabled``
+    - Enable or disable provisioning the provider via MicroProfile.
+    - N
+    - ``false``
+  * - ``dataverse.auth.oidc.client-id``
+    - The client-id of the application to identify it at your provider.
+    - Y
+    - \-
+  * - ``dataverse.auth.oidc.client-secret``
+    - A confidential secret to authorize application requests to the provider as legit.
+    - N
+    - \-
+  * - ``dataverse.auth.oidc.auth-server-url``
+    - The base URL of the OpenID Connect (OIDC) server as explained above.
+    - Y
+    - \-
+  * - ``dataverse.auth.oidc.title``
+    - The UI visible name for this provider in login options.
+    - N
+    - ``OpenID Connect``
+  * - ``dataverse.auth.oidc.subtitle``
+    - A subtitle, currently not displayed by the UI.
+    - N
+    - ``OpenID Connect``
 

From c35133adf8fbcd16ed0fbc28bb6bdcb82765083f Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Jan 2023 21:41:54 +0100
Subject: [PATCH 029/922] build(tests): fix copying all test resources as
 default missing #9268

---
 pom.xml | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/pom.xml b/pom.xml
index a26071d253b..42edbf11d8e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -598,25 +598,13 @@
     <build>
         <testResources>
             <testResource>
-                <directory>${project.basedir}/conf</directory>
+                <directory>${project.basedir}/src/test/resources</directory>
             </testResource>
-        </testResources>
-        <!--        <testResources>
-            <testResource>
-                <directory>${project.basedir}/src/main/resources</directory>
-            </testResource>
-        </testResources>-->
-        <!--        <testResources>
             <testResource>
-              <directory>${project.basedir}/src/test/java</directory>
-              <excludes>
-                  <exclude>**/*.java</exclude>
-              </excludes>
+                <directory>${project.basedir}/conf/keycloak</directory>
+                <targetPath>keycloak</targetPath>
             </testResource>
-            <testResource>
-              <directory>${project.basedir}/src/test/resources</directory>
-            </testResource>
-          </testResources>-->
+        </testResources>
         <resources>
             <resource>
                 <directory>src/main/java</directory>

From 4d7df9c4abfdebe4b8d19382fd836ff9827f5053 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 23 Feb 2023 11:13:17 -0500
Subject: [PATCH 030/922] (draft/work in progress) framework for a new file
 creation command and storage quota enforcement #9361.

---
 .../iq/dataverse/DataFileServiceBean.java     |  57 ++
 .../iq/dataverse/DatasetServiceBean.java      |   2 +-
 .../iq/dataverse/EditDatafilesPage.java       |  30 +-
 .../impl/CreateNewDataFilesCommand.java       | 555 ++++++++++++++++++
 .../settings/SettingsServiceBean.java         |  10 +-
 .../harvard/iq/dataverse/util/FileUtil.java   |  36 +-
 .../iq/dataverse/util/SystemConfig.java       |  16 +-
 .../util/bagit/data/FileUtilWrapper.java      |   7 +-
 .../FileExceedsStorageQuotaException.java     |  22 +
 src/main/java/propertyFiles/Bundle.properties |   1 +
 src/main/webapp/editFilesFragment.xhtml       |   1 +
 11 files changed, 718 insertions(+), 19 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 7da06f36be4..328f2aa59c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -67,6 +67,8 @@ public class DataFileServiceBean implements java.io.Serializable {
 
     @EJB EmbargoServiceBean embargoService;
     
+    @EJB SystemConfig systemConfig;
+    
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
     
@@ -140,6 +142,36 @@ public class DataFileServiceBean implements java.io.Serializable {
      */
     public static final String MIME_TYPE_PACKAGE_FILE = "application/vnd.dataverse.file-package";
     
+    public class UserStorageQuota {
+        private Long totalAllocatedInBytes = 0L; 
+        private Long totalUsageInBytes = 0L;
+        
+        public UserStorageQuota(Long allocated, Long used) {
+            this.totalAllocatedInBytes = allocated;
+            this.totalUsageInBytes = used; 
+        }
+        
+        public Long getTotalAllocatedInBytes() {
+            return totalAllocatedInBytes;
+        }
+        
+        public void setTotalAllocatedInBytes(Long totalAllocatedInBytes) {
+            this.totalAllocatedInBytes = totalAllocatedInBytes;
+        }
+        
+        public Long getTotalUsageInBytes() {
+            return totalUsageInBytes;
+        }
+        
+        public void setTotalUsageInBytes(Long totalUsageInBytes) {
+            this.totalUsageInBytes = totalUsageInBytes;
+        }
+        
+        public Long getRemainingQuotaInBytes() {
+            return totalAllocatedInBytes - totalUsageInBytes;
+        }
+    }
+    
     public DataFile find(Object pk) {
         return em.find(DataFile.class, pk);
     }   
@@ -1657,4 +1689,29 @@ public Embargo findEmbargo(Long id) {
         DataFile d = find(id);
         return d.getEmbargo();
     }
+    
+    public Long getStorageUsageByCreator(AuthenticatedUser user) {
+        Query query = em.createQuery("SELECT SUM(o.filesize) FROM DataFile o WHERE o.creator.id=:creatorId");
+        
+        try {
+            Long totalSize = (Long)query.setParameter("creatorId", user.getId()).getSingleResult();
+            logger.info("total size for user: "+totalSize);
+            return totalSize == null ? 0L : totalSize; 
+        } catch (NoResultException nre) { // ?
+            logger.info("NoResultException, returning 0L");
+            return 0L;
+        }
+    }
+    
+    public UserStorageQuota getUserStorageQuota(AuthenticatedUser user, Dataset dataset) {
+        // this is for testing only - one pre-set, installation-wide quota limit
+        // for everybody:
+        Long totalAllocated = systemConfig.getTestStorageQuotaLimit();
+        // again, this is for testing only - we are only counting the total size
+        // of all the files created by this user; it will likely be a much more 
+        // complex calculation in real life applications:
+        Long totalUsed = getStorageUsageByCreator(user); 
+        
+        return new UserStorageQuota(totalAllocated, totalUsed);
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
index 91ec050fe5c..4e522bbd441 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
@@ -1166,5 +1166,5 @@ public void deleteHarvestedDataset(Dataset dataset, DataverseRequest request, Lo
             hdLogger.warning("Failed to destroy the dataset");
         }
     }
-
+    
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index 1c033b37872..74c4e782d56 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -28,6 +28,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.RequestRsyncScriptCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetThumbnailCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand;
 import edu.harvard.iq.dataverse.ingest.IngestRequest;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.ingest.IngestUtil;
@@ -187,7 +188,13 @@ public enum Referrer {
     // Used to store results of permissions checks
     private final Map<String, Boolean> datasetPermissionMap = new HashMap<>(); // { Permission human_name : Boolean }
 
+    // Size limit of an individual file: (set for the storage volume used)
     private Long maxFileUploadSizeInBytes = null;
+    // Total amount of data that the user should be allowed to upload.
+    // Will be calculated in real time based on various level quotas - 
+    // for this user and/or this collection/dataset, etc. We should 
+    // assume that it may change during the user session.
+    private Long maxTotalUploadSizeInBytes = null;
     private Long maxIngestSizeInBytes = null;
     // CSV: 4.8 MB, DTA: 976.6 KB, XLSX: 5.7 MB, etc.
     private String humanPerFormatTabularLimits = null;
@@ -336,6 +343,14 @@ public Long getMaxFileUploadSizeInBytes() {
     public String getHumanMaxFileUploadSizeInBytes() {
         return FileSizeChecker.bytesToHumanReadable(this.maxFileUploadSizeInBytes);
     }
+    
+    public Long getMaxTotalUploadSizeInBytes() {
+        return maxTotalUploadSizeInBytes;
+    }
+    
+    public String getHumanMaxTotalUploadSizeInBytes() {
+        return FileSizeChecker.bytesToHumanReadable(maxTotalUploadSizeInBytes);
+    }
 
     public boolean isUnlimitedUploadFileSize() {
 
@@ -563,7 +578,6 @@ public String init() {
         this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
         this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
         this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit();        
-        this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
         
         hasValidTermsOfAccess = isHasValidTermsOfAccess();
         if (!hasValidTermsOfAccess) {
@@ -2024,7 +2038,13 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
             // Note: A single uploaded file may produce multiple datafiles - 
             // for example, multiple files can be extracted from an uncompressed
             // zip file.
-            CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig);
+            ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig);
+            
+            Command<CreateDataFileResult> cmd;
+            cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null);
+            CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
+
+        
             dFileList = createDataFilesResult.getDataFiles();
             String createDataFilesError = editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult);
             if(createDataFilesError != null) {
@@ -2033,8 +2053,14 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
             }
 
         } catch (IOException ioex) {
+            // shouldn't we try and communicate to the user what happened?
             logger.warning("Failed to process and/or save the file " + uFile.getFileName() + "; " + ioex.getMessage());
             return;
+        } catch (CommandException cex) {
+            // shouldn't we try and communicate to the user what happened?
+            errorMessages.add(cex.getMessage());
+            uploadComponentId = event.getComponent().getClientId();
+            return;
         }
         /*catch (FileExceedsMaxSizeException ex) {
             logger.warning("Failed to process and/or save the file " + uFile.getFileName() + "; " + ex.getMessage());
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
new file mode 100644
index 00000000000..9f281f9446d
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -0,0 +1,555 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
+import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
+import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
+import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
+import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
+import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.FileUtil;
+import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
+import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport;
+import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType;
+import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension;
+import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory;
+import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile;
+import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType;
+import edu.harvard.iq.dataverse.util.ShapefileHandler;
+import edu.harvard.iq.dataverse.util.StringUtil;
+import edu.harvard.iq.dataverse.util.file.BagItFileHandler;
+import edu.harvard.iq.dataverse.util.file.BagItFileHandlerFactory;
+import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import javax.enterprise.inject.spi.CDI;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ *
+ * @author landreev
+ */
+@RequiredPermissions( Permission.EditDataset )
+public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileResult> {
+    private static final Logger logger = Logger.getLogger(CreateNewDataFilesCommand.class.getCanonicalName());
+    
+    private final DatasetVersion version;
+    private final InputStream inputStream;
+    private final String fileName;
+    private final String suppliedContentType; 
+    private final String newStorageIdentifier; 
+    private final String newCheckSum; 
+    private DataFile.ChecksumType newCheckSumType;
+
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum) {
+        this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, null);
+    }
+    
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, DataFile.ChecksumType newCheckSumType) {
+        super(aRequest, version.getDataset());
+        
+        this.version = version;
+        this.inputStream = inputStream;
+        this.fileName = fileName;
+        this.suppliedContentType = suppliedContentType; 
+        this.newStorageIdentifier = newStorageIdentifier; 
+        this.newCheckSum = newCheckSum; 
+        this.newCheckSumType = newCheckSumType;
+    }
+
+    @Override
+    public CreateDataFileResult execute(CommandContext ctxt) throws CommandException {
+        List<DataFile> datafiles = new ArrayList<>();
+
+        //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default
+        if(newCheckSumType == null) {
+            newCheckSumType = ctxt.systemConfig().getFileFixityChecksumAlgorithm();
+        }
+
+        String warningMessage = null;
+
+        // save the file, in the temporary location for now: 
+        Path tempFile = null;
+
+        Long fileSizeLimit = ctxt.systemConfig().getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId());
+        Long storageQuotaLimit = null; 
+        
+        if (ctxt.systemConfig().isStorageQuotasEnforced()) {
+            //storageQuotaLimit = ctxt.files().getClass()...;
+            UserStorageQuota quota = ctxt.files().getUserStorageQuota(super.getRequest().getAuthenticatedUser(), this.version.getDataset());
+            if (quota != null) {
+                storageQuotaLimit = quota.getRemainingQuotaInBytes();
+            }
+        }
+        String finalType = null;
+        
+        if (newStorageIdentifier == null) {
+            if (getFilesTempDirectory() != null) {
+                try {
+                    tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
+                    // "temporary" location is the key here; this is why we are not using
+                    // the DataStore framework for this - the assumption is that
+                    // temp files will always be stored on the local filesystem.
+                    // -- L.A. Jul. 2014
+                    logger.fine("Will attempt to save the file as: " + tempFile.toString());
+                    Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING);
+                } catch (IOException ioex) {
+                    throw new CommandExecutionException("Failed to save the upload as a temp file (temp disk space?)", ioex, this);
+                }
+
+                // A file size check, before we do anything else:
+                // (note that "no size limit set" = "unlimited")
+                // (also note, that if this is a zip file, we'll be checking
+                // the size limit for each of the individual unpacked files)
+                Long fileSize = tempFile.toFile().length();
+                if (fileSizeLimit != null && fileSize > fileSizeLimit) {
+                    try {
+                        tempFile.toFile().delete();
+                    } catch (Exception ex) {
+                        // ignore - but log a warning
+                        logger.warning("Could not remove temp file " + tempFile.getFileName());
+                    }
+                    throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)), this);
+                }
+
+            } else {
+                throw new CommandExecutionException("Temp directory is not configured.", this);
+            }
+            
+            logger.fine("mime type supplied: " + suppliedContentType);
+            
+            // Let's try our own utilities (Jhove, etc.) to determine the file type
+            // of the uploaded file. (We may already have a mime type supplied for this
+            // file - maybe the type that the browser recognized on upload; or, if
+            // it's a harvest, maybe the remote server has already given us the type
+            // for this file... with our own type utility we may or may not do better
+            // than the type supplied:
+            // -- L.A.
+            String recognizedType = null;
+
+            try {
+                recognizedType = determineFileType(tempFile.toFile(), fileName);
+                logger.fine("File utility recognized the file as " + recognizedType);
+                if (recognizedType != null && !recognizedType.equals("")) {
+                    if (useRecognizedType(suppliedContentType, recognizedType)) {
+                        finalType = recognizedType;
+                    }
+                }
+
+            } catch (Exception ex) {
+                logger.warning("Failed to run the file utility mime type check on file " + fileName);
+            }
+
+            if (finalType == null) {
+                finalType = (suppliedContentType == null || suppliedContentType.equals(""))
+                        ? MIME_TYPE_UNDETERMINED_DEFAULT
+                        : suppliedContentType;
+            }
+
+            // A few special cases:
+            // if this is a gzipped FITS file, we'll uncompress it, and ingest it as
+            // a regular FITS file:
+            if (finalType.equals("application/fits-gzipped")) {
+
+                InputStream uncompressedIn = null;
+                String finalFileName = fileName;
+                // if the file name had the ".gz" extension, remove it,
+                // since we are going to uncompress it:
+                if (fileName != null && fileName.matches(".*\\.gz$")) {
+                    finalFileName = fileName.replaceAll("\\.gz$", "");
+                }
+
+                DataFile datafile = null;
+                try {
+                    uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));
+                    File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit, storageQuotaLimit);
+                    datafile = FileUtil.createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, ctxt.systemConfig().getFileFixityChecksumAlgorithm());
+                } catch (IOException | FileExceedsMaxSizeException | FileExceedsStorageQuotaException ioex) {
+                    // it looks like we simply skip the file silently, if its uncompressed size
+                    // exceeds the limit. we should probably report this in detail instead.
+                    datafile = null;
+                } finally {
+                    if (uncompressedIn != null) {
+                        try {
+                            uncompressedIn.close();
+                        } catch (IOException e) {
+                        }
+                    }
+                }
+
+                // If we were able to produce an uncompressed file, we'll use it
+                // to create and return a final DataFile; if not, we're not going
+                // to do anything - and then a new DataFile will be created further
+                // down, from the original, uncompressed file.
+                if (datafile != null) {
+                    // remove the compressed temp file:
+                    try {
+                        tempFile.toFile().delete();
+                    } catch (SecurityException ex) {
+                        // (this is very non-fatal)
+                        logger.warning("Failed to delete temporary file " + tempFile.toString());
+                    }
+
+                    datafiles.add(datafile);
+                    return CreateDataFileResult.success(fileName, finalType, datafiles);
+                }
+
+                // If it's a ZIP file, we are going to unpack it and create multiple
+                // DataFile objects from its contents:
+            } else if (finalType.equals("application/zip")) {
+
+                ZipInputStream unZippedIn = null;
+                ZipEntry zipEntry = null;
+
+                int fileNumberLimit = ctxt.systemConfig().getZipUploadFilesLimit();
+
+                try {
+                    Charset charset = null;
+                    /*
+                	TODO: (?)
+                	We may want to investigate somehow letting the user specify 
+                	the charset for the filenames in the zip file...
+                    - otherwise, ZipInputStream bails out if it encounteres a file 
+                	name that's not valid in the current charest (i.e., UTF-8, in 
+                    our case). It would be a bit trickier than what we're doing for 
+                    SPSS tabular ingests - with the lang. encoding pulldown menu - 
+                	because this encoding needs to be specified *before* we upload and
+                    attempt to unzip the file. 
+                	        -- L.A. 4.0 beta12
+                	logger.info("default charset is "+Charset.defaultCharset().name());
+                	if (Charset.isSupported("US-ASCII")) {
+                    	logger.info("charset US-ASCII is supported.");
+                    	charset = Charset.forName("US-ASCII");
+                    	if (charset != null) {
+                       	    logger.info("was able to obtain charset for US-ASCII");
+                    	}
+                    
+                	 }
+                     */
+
+                    if (charset != null) {
+                        unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset);
+                    } else {
+                        unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));
+                    }
+
+                    Long storageQuotaLimitForUnzippedFiles = storageQuotaLimit; 
+                    while (true) {
+                        try {
+                            zipEntry = unZippedIn.getNextEntry();
+                        } catch (IllegalArgumentException iaex) {
+                            // Note:
+                            // ZipInputStream documentation doesn't even mention that
+                            // getNextEntry() throws an IllegalArgumentException!
+                            // but that's what happens if the file name of the next
+                            // entry is not valid in the current CharSet.
+                            // -- L.A.
+                            warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is.";
+                            logger.warning(warningMessage);
+                            throw new IOException();
+                        }
+
+                        if (zipEntry == null) {
+                            break;
+                        }
+                        // Note that some zip entries may be directories - we
+                        // simply skip them:
+
+                        if (!zipEntry.isDirectory()) {
+                            if (datafiles.size() > fileNumberLimit) {
+                                logger.warning("Zip upload - too many files.");
+                                warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
+                                        + "); please upload a zip archive with fewer files, if you want them to be ingested "
+                                        + "as individual DataFiles.";
+                                throw new IOException();
+                            }
+
+                            String fileEntryName = zipEntry.getName();
+                            logger.fine("ZipEntry, file: " + fileEntryName);
+
+                            if (fileEntryName != null && !fileEntryName.equals("")) {
+
+                                String shortName = fileEntryName.replaceFirst("^.*[\\/]", "");
+
+                                // Check if it's a "fake" file - a zip archive entry
+                                // created for a MacOS X filesystem element: (these
+                                // start with "._")
+                                if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
+                                    // OK, this seems like an OK file entry - we'll try
+                                    // to read it and create a DataFile with it:
+
+                                    File unZippedTempFile = saveInputStreamInTempFile(unZippedIn, fileSizeLimit, storageQuotaLimitForUnzippedFiles);
+                                    DataFile datafile = FileUtil.createSingleDataFile(version, 
+                                            unZippedTempFile, 
+                                            null, 
+                                            shortName,
+                                            MIME_TYPE_UNDETERMINED_DEFAULT,
+                                            ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false);
+                                    
+                                    storageQuotaLimitForUnzippedFiles = storageQuotaLimitForUnzippedFiles - datafile.getFilesize();
+
+                                    if (!fileEntryName.equals(shortName)) {
+                                        // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
+                                        // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all 
+                                        // the leading, trailing and duplicate slashes; then replace all the characters that 
+                                        // don't pass our validation rules.
+                                        String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", "");
+                                        directoryName = StringUtil.sanitizeFileDirectory(directoryName, true);
+                                        // if (!"".equals(directoryName)) {
+                                        if (!StringUtil.isEmpty(directoryName)) {
+                                            logger.fine("setting the directory label to " + directoryName);
+                                            datafile.getFileMetadata().setDirectoryLabel(directoryName);
+                                        }
+                                    }
+
+                                    if (datafile != null) {
+                                        // We have created this datafile with the mime type "unknown";
+                                        // Now that we have it saved in a temporary location,
+                                        // let's try and determine its real type:
+
+                                        String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
+
+                                        try {
+                                            recognizedType = determineFileType(new File(tempFileName), shortName);
+                                            logger.fine("File utility recognized unzipped file as " + recognizedType);
+                                            if (recognizedType != null && !recognizedType.equals("")) {
+                                                datafile.setContentType(recognizedType);
+                                            }
+                                        } catch (Exception ex) {
+                                            logger.warning("Failed to run the file utility mime type check on file " + fileName);
+                                        }
+
+                                        datafiles.add(datafile);
+                                    }
+                                }
+                            }
+                        }
+                        unZippedIn.closeEntry();
+
+                    }
+
+                } catch (IOException ioex) {
+                    // just clear the datafiles list and let
+                    // ingest default to creating a single DataFile out
+                    // of the unzipped file.
+                    logger.warning("Unzipping failed; rolling back to saving the file as is.");
+                    if (warningMessage == null) {
+                        warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed");
+                    }
+
+                    datafiles.clear();
+                } catch (FileExceedsMaxSizeException femsx) {
+                    logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage());
+                    warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.size", Arrays.asList(FileSizeChecker.bytesToHumanReadable(fileSizeLimit)));
+                    datafiles.clear();
+                } catch (FileExceedsStorageQuotaException fesqx) {
+                    logger.warning("One of the unzipped files exceeds the storage quota limit; resorting to saving the file as is. " + fesqx.getMessage());
+                    warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));
+                    datafiles.clear();
+                } finally {
+                    if (unZippedIn != null) {
+                        try {
+                            unZippedIn.close();
+                        } catch (Exception zEx) {
+                        }
+                    }
+                }
+                if (datafiles.size() > 0) {
+                    // remove the uploaded zip file:
+                    try {
+                        Files.delete(tempFile);
+                    } catch (IOException ioex) {
+                        // do nothing - it's just a temp file.
+                        logger.warning("Could not remove temp file " + tempFile.getFileName().toString());
+                    }
+                    // and return:
+                    return CreateDataFileResult.success(fileName, finalType, datafiles);
+                }
+
+            } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) {
+                // Shape files may have to be split into multiple files,
+                // one zip archive per each complete set of shape files:
+
+                // File rezipFolder = new File(this.getFilesTempDirectory());
+                File rezipFolder = FileUtil.getShapefileUnzipTempDirectory();
+
+                IngestServiceShapefileHelper shpIngestHelper;
+                shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder);
+
+                boolean didProcessWork = shpIngestHelper.processFile();
+                if (!(didProcessWork)) {
+                    logger.severe("Processing of zipped shapefile failed.");
+                    return CreateDataFileResult.error(fileName, finalType);
+                }
+
+                try {
+                    Long storageQuotaLimitForRezippedFiles = storageQuotaLimit;
+                    
+                    for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) {
+                        FileInputStream finalFileInputStream = new FileInputStream(finalFile);
+                        finalType = FileUtil.determineContentType(finalFile);
+                        if (finalType == null) {
+                            logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'");
+                            continue;
+                        }
+
+                        File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit, storageQuotaLimitForRezippedFiles);
+                        DataFile new_datafile = FileUtil.createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, ctxt.systemConfig().getFileFixityChecksumAlgorithm());
+                        
+                        String directoryName = null;
+                        String absolutePathName = finalFile.getParent();
+                        if (absolutePathName != null) {
+                            if (absolutePathName.length() > rezipFolder.toString().length()) {
+                                // This file lives in a subfolder - we want to 
+                                // preserve it in the FileMetadata:
+                                directoryName = absolutePathName.substring(rezipFolder.toString().length() + 1);
+
+                                if (!StringUtil.isEmpty(directoryName)) {
+                                    new_datafile.getFileMetadata().setDirectoryLabel(directoryName);
+                                }
+                            }
+                        }
+                        if (new_datafile != null) {
+                            datafiles.add(new_datafile);
+                            // todo: can this new_datafile be null?
+                            storageQuotaLimitForRezippedFiles = storageQuotaLimitForRezippedFiles - new_datafile.getFilesize();
+                        } else {
+                            logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName());
+                        }
+                        try {
+                            finalFileInputStream.close();
+                        } catch (IOException ioex) {
+                            // this one can be ignored
+                        }
+
+                    }
+                } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) {
+                    logger.severe("One of the unzipped shape files exceeded the size limit, or the storage quota; giving up. " + femsx.getMessage());
+                    datafiles.clear();
+                    // (or should we throw an exception, instead of skipping it quietly?
+                } catch (IOException ioex) {
+                    throw new CommandExecutionException("Failed to process one of the components of the unpacked shape file", ioex, this);
+                    // todo? - maybe try to provide a more detailed explanation, of which repackaged component, etc.?
+                }
+
+                // Delete the temp directory used for unzipping
+                // The try-catch is due to error encountered in using NFS for stocking file,
+                // cf. https://github.com/IQSS/dataverse/issues/5909
+                try {
+                    FileUtils.deleteDirectory(rezipFolder);
+                } catch (IOException ioex) {
+                    // do nothing - it's a temp folder.
+                    logger.warning("Could not remove temp folder, error message : " + ioex.getMessage());
+                }
+
+                if (datafiles.size() > 0) {
+                    // remove the uploaded zip file:
+                    try {
+                        Files.delete(tempFile);
+                    } catch (IOException ioex) {
+                        // ignore - it's just a temp file - but let's log a warning
+                        logger.warning("Could not remove temp file " + tempFile.getFileName().toString());
+                    } catch (SecurityException se) {
+                        // same
+                        logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: "
+                                + se.getMessage());
+                    }
+                    return CreateDataFileResult.success(fileName, finalType, datafiles);
+                } else {
+                    logger.severe("No files added from directory of rezipped shapefiles");
+                }
+                return CreateDataFileResult.error(fileName, finalType);
+
+            } else if (finalType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE)) {
+                
+                try { 
+                    Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
+                    if (bagItFileHandler.isPresent()) {
+                        CreateDataFileResult result = bagItFileHandler.get().handleBagItPackage(ctxt.systemConfig(), version, fileName, tempFile.toFile());
+                        return result;
+                    }
+                } catch (IOException ioex) {
+                    throw new CommandExecutionException("Failed to process uploaded BagIt file", ioex, this);
+                }
+            }
+        } else {
+            // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied
+            finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType;
+            String type = determineFileTypeByNameAndExtension(fileName);
+            if (!StringUtils.isBlank(type)) {
+                //Use rules for deciding when to trust browser supplied type
+                if (useRecognizedType(finalType, type)) {
+                    finalType = type;
+                }
+                logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType);
+            }
+        }
+        // Finally, if none of the special cases above were applicable (or 
+        // if we were unable to unpack an uploaded file, etc.), we'll just 
+        // create and return a single DataFile:
+        File newFile = null;
+        if (tempFile != null) {
+            newFile = tempFile.toFile();
+        }
+        
+        // We have already checked that this file does not exceed the individual size limit; 
+        // but if we are processing it as is, as a single file, we need to check if 
+        // its size does not go beyond the allocated storage quota (if specified):
+        
+        long fileSize = newFile.length();
+        
+        if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
+            try {
+                tempFile.toFile().delete();
+            } catch (Exception ex) {
+                // ignore - but log a warning
+                logger.warning("Could not remove temp file " + tempFile.getFileName());
+            }
+            throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)), this);
+        } 
+        
+        DataFile datafile = FileUtil.createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum);
+        File f = null;
+        if (tempFile != null) {
+            f = tempFile.toFile();
+        }
+        if (datafile != null && ((f != null) || (newStorageIdentifier != null))) {
+
+            if (warningMessage != null) {
+                createIngestFailureReport(datafile, warningMessage);
+                datafile.SetIngestProblem();
+            }
+            datafiles.add(datafile);
+
+            return CreateDataFileResult.success(fileName, finalType, datafiles);
+        }
+
+        return CreateDataFileResult.error(fileName, finalType);
+    }   // end createDataFiles
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
index d84e18d5931..7f44b4c6a0d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
@@ -576,7 +576,15 @@ Whether Harvesting (OAI) service is enabled
         /**
          * The URL for the DvWebLoader tool (see github.com/gdcc/dvwebloader for details)
          */
-        WebloaderUrl
+        WebloaderUrl, 
+        /**
+         * Enforce storage quotas:
+         */
+        UseStorageQuotas, 
+        /** 
+         * Placeholder storage quota (defines the same quota setting for every user; used to test the concept of a quota.
+         */
+        StorageQuotaSizeInBytes
 
         ;
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index c600abfd409..ba24472b314 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -105,6 +105,7 @@
 
 import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
 import java.util.Arrays;
 import org.apache.commons.io.IOUtils;
@@ -411,7 +412,7 @@ public static String getUserFriendlyOriginalType(DataFile dataFile) {
      *  Returns a content type string for a FileObject
      * 
      */
-    private static String determineContentType(File fileObject) {
+    public static String determineContentType(File fileObject) {
         if (fileObject==null){
             return null;
         }
@@ -902,7 +903,7 @@ public static CreateDataFileResult createDataFiles(DatasetVersion version, Input
                     uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));
                     File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit);
                     datafile = createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, systemConfig.getFileFixityChecksumAlgorithm());
-                } catch (IOException | FileExceedsMaxSizeException ioex) {
+                } catch (IOException | FileExceedsMaxSizeException | FileExceedsStorageQuotaException ioex) {
                     datafile = null;
                 } finally {
                     if (uncompressedIn != null) {
@@ -1068,7 +1069,7 @@ public static CreateDataFileResult createDataFiles(DatasetVersion version, Input
                     }
 
                     datafiles.clear();
-                } catch (FileExceedsMaxSizeException femsx) {
+                } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) {
                     logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage());
                     warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.size", Arrays.asList(FileSizeChecker.bytesToHumanReadable(fileSizeLimit)));
                     datafiles.clear();
@@ -1154,7 +1155,7 @@ public static CreateDataFileResult createDataFiles(DatasetVersion version, Input
                         finalFileInputStream.close();
 
                     }
-                } catch (FileExceedsMaxSizeException femsx) {
+                } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) {
                     logger.severe("One of the unzipped shape files exceeded the size limit; giving up. " + femsx.getMessage());
                     datafiles.clear();
                 }
@@ -1271,7 +1272,12 @@ public static boolean useRecognizedType(String suppliedContentType, String recog
 	}
 
 	public static File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit)
-            throws IOException, FileExceedsMaxSizeException {
+            throws IOException, FileExceedsMaxSizeException, FileExceedsStorageQuotaException {
+            return saveInputStreamInTempFile(inputStream, fileSizeLimit, null);
+        }
+        
+        public static File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit, Long storageQuotaLimit)
+            throws IOException, FileExceedsMaxSizeException, FileExceedsStorageQuotaException {
         Path tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
         
         if (inputStream != null && tempFile != null) {
@@ -1285,6 +1291,11 @@ public static File saveInputStreamInTempFile(InputStream inputStream, Long fileS
                 throw new FileExceedsMaxSizeException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)));  
             }
             
+            if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
+                try {tempFile.toFile().delete();} catch (Exception ex) {}
+                throw new FileExceedsStorageQuotaException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_quota"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)));  
+            }
+            
             return tempFile.toFile();
         }
         throw new IOException("Failed to save uploaded file.");
@@ -1325,7 +1336,6 @@ public static DataFile createSingleDataFile(DatasetVersion version, File tempFil
         datafile.setPermissionModificationTime(new Timestamp(new Date().getTime()));
         FileMetadata fmd = new FileMetadata();
 
-        // TODO: add directoryLabel?
         fmd.setLabel(fileName);
 
         if (addToDataset) {
@@ -1341,13 +1351,13 @@ public static DataFile createSingleDataFile(DatasetVersion version, File tempFil
             fmd.setDatasetVersion(version);
             version.getDataset().getFiles().add(datafile);
         }
-        if(storageIdentifier==null) {
-        generateStorageIdentifier(datafile);
-        if (!tempFile.renameTo(new File(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier()))) {
-            return null;
-        }
+        if (storageIdentifier == null) {
+            generateStorageIdentifier(datafile);
+            if (!tempFile.renameTo(new File(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier()))) {
+                return null;
+            }
         } else {
-        	datafile.setStorageIdentifier(storageIdentifier);
+            datafile.setStorageIdentifier(storageIdentifier);
         }
 
         if ((checksum !=null)&&(!checksum.isEmpty())) {
@@ -1372,7 +1382,7 @@ public static DataFile createSingleDataFile(DatasetVersion version, File tempFil
     
         Naming convention: getFilesTempDirectory() + "shp_" + "yyyy-MM-dd-hh-mm-ss-SSS"
     */
-    private static File getShapefileUnzipTempDirectory(){
+    public static File getShapefileUnzipTempDirectory(){
         
         String tempDirectory = getFilesTempDirectory();
         if (tempDirectory == null){
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index c989add6e3d..ac4a3970379 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -892,7 +892,7 @@ public String toString() {
         }
 
     }
-
+    
     public boolean isPublicInstall(){
         boolean saneDefault = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, saneDefault);
@@ -1149,4 +1149,18 @@ public boolean isSignupDisabledForRemoteAuthProvider(String providerId) {
         
         return !ret; 
     }
+    
+    public boolean isStorageQuotasEnforced() {
+        return settingsService.isTrueForKey(SettingsServiceBean.Key.UseStorageQuotas, false);
+    }
+    
+    /**
+     * This method should only be used for testing of the new storage quota 
+     * mechanism, temporarily. (it uses the same value as the quota for 
+     * *everybody* regardless of the circumstances, defined as a database 
+     * setting)
+     */
+    public Long getTestStorageQuotaLimit() {
+        return settingsService.getValueForKeyAsLong(SettingsServiceBean.Key.StorageQuotaSizeInBytes);
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java
index 2bcac04076a..ecb34bdcfb5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/data/FileUtilWrapper.java
@@ -3,6 +3,7 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
+import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.util.FileUtil;
 
 import java.io.File;
@@ -43,7 +44,11 @@ public void deleteFile(Path filePath) {
     }
 
     public File saveInputStreamInTempFile(InputStream inputStream, Long fileSizeLimit) throws IOException, FileExceedsMaxSizeException {
-        return FileUtil.saveInputStreamInTempFile(inputStream, fileSizeLimit);
+        try {
+            return FileUtil.saveInputStreamInTempFile(inputStream, fileSizeLimit);
+        } catch (FileExceedsStorageQuotaException fesqx) {
+            return null; 
+        } 
     }
 
     public String determineFileType(File file, String fileName) throws IOException {
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java b/src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java
new file mode 100644
index 00000000000..29eeca254f7
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/util/file/FileExceedsStorageQuotaException.java
@@ -0,0 +1,22 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package edu.harvard.iq.dataverse.util.file;
+
+/**
+ *
+ * @author landreev
+ */
+public class FileExceedsStorageQuotaException extends Exception {
+
+    public FileExceedsStorageQuotaException(String message) {
+        super(message);
+    }
+
+    public FileExceedsStorageQuotaException(String message, Throwable cause) {
+        super(message, cause);
+    }
+    
+}
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 45807dc7cde..c1fd4ebaf10 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2145,6 +2145,7 @@ file.message.replaceSuccess=The file has been replaced.
 file.addreplace.file_size_ok=File size is in range.
 file.addreplace.error.byte_abrev=B
 file.addreplace.error.file_exceeds_limit=This file size ({0}) exceeds the size limit of {1}.
+file.addreplace.error.quota_exceeded=This file (size {0}) exceeds the remaining storage quota of {1}.
 file.addreplace.error.dataset_is_null=The dataset cannot be null.
 file.addreplace.error.dataset_id_is_null=The dataset ID cannot be null.
 file.addreplace.error.parsing=Error in parsing provided json
diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml
index a4e635b8c14..99db5abd2dc 100644
--- a/src/main/webapp/editFilesFragment.xhtml
+++ b/src/main/webapp/editFilesFragment.xhtml
@@ -155,6 +155,7 @@
                                   fileLimit="#{EditDatafilesPage.getMaxNumberOfFiles()}" 
                                   invalidSizeMessage="#{bundle['file.edit.error.file_exceeds_limit']}" 
                                   sequential="true"
+                                  previewWidth="-1"
                                   widgetVar="fileUploadWidget">
                         <f:passThroughAttribute name="aria-label" value="#{bundle['file.uploadFiles']}"/>
                     </p:fileUpload>

From d5fd5e3e690f0b39d630b4774ec2807b2ec08750 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 23 Feb 2023 18:35:56 -0500
Subject: [PATCH 031/922] switched to the new Create Files command in the
 remaining places where the utility was used. #9361

---
 .../iq/dataverse/EditDatafilesPage.java       | 18 +++---
 .../datadeposit/MediaResourceManagerImpl.java | 58 ++++++++++---------
 .../datasetutility/AddReplaceFileHelper.java  | 12 ++--
 .../harvard/iq/dataverse/util/FileUtil.java   |  6 +-
 4 files changed, 54 insertions(+), 40 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index 74c4e782d56..928bf635ffa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -1508,14 +1508,16 @@ public void handleDropBoxUpload(ActionEvent event) {
                 // for example, multiple files can be extracted from an uncompressed
                 // zip file.
                 //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");
-                CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig);
+                //CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig);
+                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null);
+                CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                 datafiles = createDataFilesResult.getDataFiles();
                 Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
 
-            } catch (IOException ex) {
+            } catch (CommandException ex) {
                 this.logger.log(Level.SEVERE, "Error during ingest of DropBox file {0} from link {1}", new Object[]{fileName, fileLink});
                 continue;
-            }/*catch (FileExceedsMaxSizeException ex){
+            } /*catch (FileExceedsMaxSizeException ex){
                 this.logger.log(Level.SEVERE, "Error during ingest of DropBox file {0} from link {1}: {2}", new Object[]{fileName, fileLink, ex.getMessage()});
                 continue;
             }*/ finally {
@@ -2040,8 +2042,7 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
             // zip file.
             ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig);
             
-            Command<CreateDataFileResult> cmd;
-            cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null);
+            Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null);
             CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
 
         
@@ -2165,10 +2166,13 @@ public void handleExternalUpload() {
                     // for example, multiple files can be extracted from an uncompressed
                     // zip file.
                     //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");
-                    CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig);
+                    ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig);
+                    
+                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType);
+                    CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                     datafiles = createDataFilesResult.getDataFiles();
                     Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
-                } catch (IOException ex) {
+                } catch (CommandException ex) {
                     logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName});
                 }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
index 5491024c73c..f21a65bdf1e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
@@ -6,14 +6,17 @@
 import edu.harvard.iq.dataverse.DatasetServiceBean;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
 import edu.harvard.iq.dataverse.PermissionServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
+import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -69,6 +72,8 @@ public class MediaResourceManagerImpl implements MediaResourceManager {
     SwordAuth swordAuth;
     @Inject
     UrlManager urlManager;
+    @Inject
+    DataverseRequestServiceBean dvRequestService;
 
     private HttpServletRequest httpRequest;
 
@@ -301,37 +306,38 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au
              */
             String guessContentTypeForMe = null;
             List<DataFile> dataFiles = new ArrayList<>();
+
             try {
-                try {
-                    CreateDataFileResult createDataFilesResponse =  FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig);
-                    dataFiles = createDataFilesResponse.getDataFiles();
-                } catch (EJBException ex) {
-                    Throwable cause = ex.getCause();
-                    if (cause != null) {
-                        if (cause instanceof IllegalArgumentException) {
-                            /**
-                             * @todo should be safe to remove this catch of
-                             * EJBException and IllegalArgumentException once
-                             * this ticket is resolved:
-                             *
-                             * IllegalArgumentException: MALFORMED when
-                             * uploading certain zip files
-                             * https://github.com/IQSS/dataverse/issues/1021
-                             */
-                            throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles. Problem with zip file, perhaps: " + cause);
-                        } else {
-                            throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles: " + cause);
-                        }
+                //CreateDataFileResult createDataFilesResponse =  FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig);
+                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null);
+                CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
+                dataFiles = createDataFilesResult.getDataFiles();
+            } catch (CommandException ex) {
+                Throwable cause = ex.getCause();
+                if (cause != null) {
+                    if (cause instanceof IllegalArgumentException) {
+                        /**
+                         * @todo should be safe to remove this catch of
+                         * EJBException and IllegalArgumentException once this
+                         * ticket is resolved:
+                         *
+                         * IllegalArgumentException: MALFORMED when uploading
+                         * certain zip files
+                         * https://github.com/IQSS/dataverse/issues/1021
+                         */
+                        throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset. Problem with zip file, perhaps: " + cause);
                     } else {
-                        throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles. No cause: " + ex.getMessage());
+                        throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + cause);
                     }
-                } /*TODO: L.A. 4.6! catch (FileExceedsMaxSizeException ex) {
+                } else {
+                    throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + ex.getMessage());
+                }
+            }
+            /*TODO: L.A. 4.6! catch (FileExceedsMaxSizeException ex) {
                     throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Exception caught calling ingestService.createDataFiles: " + ex.getMessage());
                     //Logger.getLogger(MediaResourceManagerImpl.class.getName()).log(Level.SEVERE, null, ex);
-                }*/
-            } catch (IOException ex) {
-                throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + ex.getMessage());
-            }
+            }*/
+            
             if (!dataFiles.isEmpty()) {
                 Set<ConstraintViolation> constraintViolations = editVersion.validate();
                 if (constraintViolations.size() > 0) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
index 1d0ec0f19d9..e31f86093ed 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -63,6 +63,7 @@
 
 import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_ERROR;
 import static edu.harvard.iq.dataverse.api.AbstractApiBean.STATUS_OK;
+import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand;
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 
 /**
@@ -1205,17 +1206,20 @@ private boolean step_030_createNewFilesViaIngest(){
             clone = workingVersion.cloneDatasetVersion();
         }
         try {
-            CreateDataFileResult result = FileUtil.createDataFiles(workingVersion,
+            /*CreateDataFileResult result = FileUtil.createDataFiles(workingVersion,
                     this.newFileInputStream,
                     this.newFileName,
                     this.newFileContentType,
                     this.newStorageIdentifier,
                     this.newCheckSum,
                     this.newCheckSumType,
-                    this.systemConfig);
-            initialFileList = result.getDataFiles();
+                    this.systemConfig);*/
+            
+            Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, newCheckSum, newCheckSumType);
+            CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
+            initialFileList = createDataFilesResult.getDataFiles();
 
-        } catch (IOException ex) {
+        } catch (CommandException ex) {
             if (!Strings.isNullOrEmpty(ex.getMessage())) {
                 this.addErrorSevere(getBundleErr("ingest_create_file_err") + " " + ex.getMessage());
             } else {
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index ba24472b314..0c099242849 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -805,7 +805,7 @@ public static String generateOriginalExtension(String fileType) {
         return "";
     }
     
-    public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream,
+    /*public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream,
             String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum,
             SystemConfig systemConfig)  throws IOException {
         ChecksumType checkSumType = DataFile.ChecksumType.MD5;
@@ -813,7 +813,7 @@ public static CreateDataFileResult createDataFiles(DatasetVersion version, Input
             checkSumType = systemConfig.getFileFixityChecksumAlgorithm();
         }
         return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig);
-    }
+    }*/
     
     public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException {
         List<DataFile> datafiles = new ArrayList<>();
@@ -1293,7 +1293,7 @@ public static File saveInputStreamInTempFile(InputStream inputStream, Long fileS
             
             if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
                 try {tempFile.toFile().delete();} catch (Exception ex) {}
-                throw new FileExceedsStorageQuotaException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_quota"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)));  
+                throw new FileExceedsStorageQuotaException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)));  
             }
             
             return tempFile.toFile();

From 6210c3435ff7df308a6491c5b9a0b0b23d758774 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 23 Feb 2023 18:43:23 -0500
Subject: [PATCH 032/922] removed the static utility methods that have been
 turned into a command. #9361

---
 .../harvard/iq/dataverse/util/FileUtil.java   | 430 ------------------
 1 file changed, 430 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 0c099242849..014f44c5c33 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -804,436 +804,6 @@ public static String generateOriginalExtension(String fileType) {
         }
         return "";
     }
-    
-    /*public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream,
-            String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum,
-            SystemConfig systemConfig)  throws IOException {
-        ChecksumType checkSumType = DataFile.ChecksumType.MD5;
-        if (newStorageIdentifier == null) {
-            checkSumType = systemConfig.getFileFixityChecksumAlgorithm();
-        }
-        return createDataFiles(version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, checkSumType, systemConfig);
-    }*/
-    
-    public static CreateDataFileResult createDataFiles(DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, ChecksumType newCheckSumType, SystemConfig systemConfig) throws IOException {
-        List<DataFile> datafiles = new ArrayList<>();
-
-        //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default
-        if(newCheckSumType == null) {
-            newCheckSumType = systemConfig.getFileFixityChecksumAlgorithm();
-        }
-
-        String warningMessage = null;
-
-        // save the file, in the temporary location for now: 
-        Path tempFile = null;
-
-        Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId());
-        String finalType = null;
-        if (newStorageIdentifier == null) {
-            if (getFilesTempDirectory() != null) {
-                tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
-                // "temporary" location is the key here; this is why we are not using
-                // the DataStore framework for this - the assumption is that
-                // temp files will always be stored on the local filesystem.
-                // -- L.A. Jul. 2014
-                logger.fine("Will attempt to save the file as: " + tempFile.toString());
-                Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING);
-
-                // A file size check, before we do anything else:
-                // (note that "no size limit set" = "unlimited")
-                // (also note, that if this is a zip file, we'll be checking
-                // the size limit for each of the individual unpacked files)
-                Long fileSize = tempFile.toFile().length();
-                if (fileSizeLimit != null && fileSize > fileSizeLimit) {
-                    try {
-                        tempFile.toFile().delete();
-                    } catch (Exception ex) {
-                    }
-                    throw new IOException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)));
-                }
-
-            } else {
-                throw new IOException("Temp directory is not configured.");
-            }
-            logger.fine("mime type supplied: " + suppliedContentType);
-            // Let's try our own utilities (Jhove, etc.) to determine the file type
-            // of the uploaded file. (We may already have a mime type supplied for this
-            // file - maybe the type that the browser recognized on upload; or, if
-            // it's a harvest, maybe the remote server has already given us the type
-            // for this file... with our own type utility we may or may not do better
-            // than the type supplied:
-            // -- L.A.
-            String recognizedType = null;
-
-            try {
-                recognizedType = determineFileType(tempFile.toFile(), fileName);
-                logger.fine("File utility recognized the file as " + recognizedType);
-                if (recognizedType != null && !recognizedType.equals("")) {
-                    if (useRecognizedType(suppliedContentType, recognizedType)) {
-                        finalType = recognizedType;
-                    }
-                }
-
-            } catch (Exception ex) {
-                logger.warning("Failed to run the file utility mime type check on file " + fileName);
-            }
-
-            if (finalType == null) {
-                finalType = (suppliedContentType == null || suppliedContentType.equals(""))
-                        ? MIME_TYPE_UNDETERMINED_DEFAULT
-                        : suppliedContentType;
-            }
-
-            // A few special cases:
-            // if this is a gzipped FITS file, we'll uncompress it, and ingest it as
-            // a regular FITS file:
-            if (finalType.equals("application/fits-gzipped")) {
-
-                InputStream uncompressedIn = null;
-                String finalFileName = fileName;
-                // if the file name had the ".gz" extension, remove it,
-                // since we are going to uncompress it:
-                if (fileName != null && fileName.matches(".*\\.gz$")) {
-                    finalFileName = fileName.replaceAll("\\.gz$", "");
-                }
-
-                DataFile datafile = null;
-                try {
-                    uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));
-                    File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit);
-                    datafile = createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, systemConfig.getFileFixityChecksumAlgorithm());
-                } catch (IOException | FileExceedsMaxSizeException | FileExceedsStorageQuotaException ioex) {
-                    datafile = null;
-                } finally {
-                    if (uncompressedIn != null) {
-                        try {
-                            uncompressedIn.close();
-                        } catch (IOException e) {
-                        }
-                    }
-                }
-
-                // If we were able to produce an uncompressed file, we'll use it
-                // to create and return a final DataFile; if not, we're not going
-                // to do anything - and then a new DataFile will be created further
-                // down, from the original, uncompressed file.
-                if (datafile != null) {
-                    // remove the compressed temp file:
-                    try {
-                        tempFile.toFile().delete();
-                    } catch (SecurityException ex) {
-                        // (this is very non-fatal)
-                        logger.warning("Failed to delete temporary file " + tempFile.toString());
-                    }
-
-                    datafiles.add(datafile);
-                    return CreateDataFileResult.success(fileName, finalType, datafiles);
-                }
-
-                // If it's a ZIP file, we are going to unpack it and create multiple
-                // DataFile objects from its contents:
-            } else if (finalType.equals("application/zip")) {
-
-                ZipInputStream unZippedIn = null;
-                ZipEntry zipEntry = null;
-
-                int fileNumberLimit = systemConfig.getZipUploadFilesLimit();
-
-                try {
-                    Charset charset = null;
-                    /*
-                	TODO: (?)
-                	We may want to investigate somehow letting the user specify 
-                	the charset for the filenames in the zip file...
-                    - otherwise, ZipInputStream bails out if it encounteres a file 
-                	name that's not valid in the current charest (i.e., UTF-8, in 
-                    our case). It would be a bit trickier than what we're doing for 
-                    SPSS tabular ingests - with the lang. encoding pulldown menu - 
-                	because this encoding needs to be specified *before* we upload and
-                    attempt to unzip the file. 
-                	        -- L.A. 4.0 beta12
-                	logger.info("default charset is "+Charset.defaultCharset().name());
-                	if (Charset.isSupported("US-ASCII")) {
-                    	logger.info("charset US-ASCII is supported.");
-                    	charset = Charset.forName("US-ASCII");
-                    	if (charset != null) {
-                       	    logger.info("was able to obtain charset for US-ASCII");
-                    	}
-                    
-                	 }
-                     */
-
-                    if (charset != null) {
-                        unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset);
-                    } else {
-                        unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));
-                    }
-
-                    while (true) {
-                        try {
-                            zipEntry = unZippedIn.getNextEntry();
-                        } catch (IllegalArgumentException iaex) {
-                            // Note:
-                            // ZipInputStream documentation doesn't even mention that
-                            // getNextEntry() throws an IllegalArgumentException!
-                            // but that's what happens if the file name of the next
-                            // entry is not valid in the current CharSet.
-                            // -- L.A.
-                            warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is.";
-                            logger.warning(warningMessage);
-                            throw new IOException();
-                        }
-
-                        if (zipEntry == null) {
-                            break;
-                        }
-                        // Note that some zip entries may be directories - we
-                        // simply skip them:
-
-                        if (!zipEntry.isDirectory()) {
-                            if (datafiles.size() > fileNumberLimit) {
-                                logger.warning("Zip upload - too many files.");
-                                warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
-                                        + "); please upload a zip archive with fewer files, if you want them to be ingested "
-                                        + "as individual DataFiles.";
-                                throw new IOException();
-                            }
-
-                            String fileEntryName = zipEntry.getName();
-                            logger.fine("ZipEntry, file: " + fileEntryName);
-
-                            if (fileEntryName != null && !fileEntryName.equals("")) {
-
-                                String shortName = fileEntryName.replaceFirst("^.*[\\/]", "");
-
-                                // Check if it's a "fake" file - a zip archive entry
-                                // created for a MacOS X filesystem element: (these
-                                // start with "._")
-                                if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
-                                    // OK, this seems like an OK file entry - we'll try
-                                    // to read it and create a DataFile with it:
-
-                                    File unZippedTempFile = saveInputStreamInTempFile(unZippedIn, fileSizeLimit);
-                                    DataFile datafile = createSingleDataFile(version, unZippedTempFile, null, shortName,
-                                            MIME_TYPE_UNDETERMINED_DEFAULT,
-                                            systemConfig.getFileFixityChecksumAlgorithm(), null, false);
-
-                                    if (!fileEntryName.equals(shortName)) {
-                                        // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
-                                        // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all 
-                                        // the leading, trailing and duplicate slashes; then replace all the characters that 
-                                        // don't pass our validation rules.
-                                        String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", "");
-                                        directoryName = StringUtil.sanitizeFileDirectory(directoryName, true);
-                                        // if (!"".equals(directoryName)) {
-                                        if (!StringUtil.isEmpty(directoryName)) {
-                                            logger.fine("setting the directory label to " + directoryName);
-                                            datafile.getFileMetadata().setDirectoryLabel(directoryName);
-                                        }
-                                    }
-
-                                    if (datafile != null) {
-                                        // We have created this datafile with the mime type "unknown";
-                                        // Now that we have it saved in a temporary location,
-                                        // let's try and determine its real type:
-
-                                        String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
-
-                                        try {
-                                            recognizedType = determineFileType(new File(tempFileName), shortName);
-                                            logger.fine("File utility recognized unzipped file as " + recognizedType);
-                                            if (recognizedType != null && !recognizedType.equals("")) {
-                                                datafile.setContentType(recognizedType);
-                                            }
-                                        } catch (Exception ex) {
-                                            logger.warning("Failed to run the file utility mime type check on file " + fileName);
-                                        }
-
-                                        datafiles.add(datafile);
-                                    }
-                                }
-                            }
-                        }
-                        unZippedIn.closeEntry();
-
-                    }
-
-                } catch (IOException ioex) {
-                    // just clear the datafiles list and let
-                    // ingest default to creating a single DataFile out
-                    // of the unzipped file.
-                    logger.warning("Unzipping failed; rolling back to saving the file as is.");
-                    if (warningMessage == null) {
-                        warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed");
-                    }
-
-                    datafiles.clear();
-                } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) {
-                    logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage());
-                    warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.size", Arrays.asList(FileSizeChecker.bytesToHumanReadable(fileSizeLimit)));
-                    datafiles.clear();
-                } finally {
-                    if (unZippedIn != null) {
-                        try {
-                            unZippedIn.close();
-                        } catch (Exception zEx) {
-                        }
-                    }
-                }
-                if (datafiles.size() > 0) {
-                    // link the data files to the dataset/version:
-                    // (except we no longer want to do this! -- 4.6)
-                    /*Iterator<DataFile> itf = datafiles.iterator();
-                	while (itf.hasNext()) {
-                    	DataFile datafile = itf.next();
-                    	datafile.setOwner(version.getDataset());
-                        if (version.getFileMetadatas() == null) {
-                        	version.setFileMetadatas(new ArrayList());
-                        }
-                    	version.getFileMetadatas().add(datafile.getFileMetadata());
-                    	datafile.getFileMetadata().setDatasetVersion(version);
-                    
-                    	version.getDataset().getFiles().add(datafile);
-                	} */
-                    // remove the uploaded zip file:
-                    try {
-                        Files.delete(tempFile);
-                    } catch (IOException ioex) {
-                        // do nothing - it's just a temp file.
-                        logger.warning("Could not remove temp file " + tempFile.getFileName().toString());
-                    }
-                    // and return:
-                    return CreateDataFileResult.success(fileName, finalType, datafiles);
-                }
-
-            } else if (finalType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)) {
-                // Shape files may have to be split into multiple files,
-                // one zip archive per each complete set of shape files:
-
-                // File rezipFolder = new File(this.getFilesTempDirectory());
-                File rezipFolder = getShapefileUnzipTempDirectory();
-
-                IngestServiceShapefileHelper shpIngestHelper;
-                shpIngestHelper = new IngestServiceShapefileHelper(tempFile.toFile(), rezipFolder);
-
-                boolean didProcessWork = shpIngestHelper.processFile();
-                if (!(didProcessWork)) {
-                    logger.severe("Processing of zipped shapefile failed.");
-                    return CreateDataFileResult.error(fileName, finalType);
-                }
-
-                try {
-                    for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) {
-                        FileInputStream finalFileInputStream = new FileInputStream(finalFile);
-                        finalType = determineContentType(finalFile);
-                        if (finalType == null) {
-                            logger.warning("Content type is null; but should default to 'MIME_TYPE_UNDETERMINED_DEFAULT'");
-                            continue;
-                        }
-
-                        File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit);
-                        DataFile new_datafile = createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, systemConfig.getFileFixityChecksumAlgorithm());
-                        String directoryName = null;
-                        String absolutePathName = finalFile.getParent();
-                        if (absolutePathName != null) {
-                            if (absolutePathName.length() > rezipFolder.toString().length()) {
-                                // This file lives in a subfolder - we want to 
-                                // preserve it in the FileMetadata:
-                                directoryName = absolutePathName.substring(rezipFolder.toString().length() + 1);
-
-                                if (!StringUtil.isEmpty(directoryName)) {
-                                    new_datafile.getFileMetadata().setDirectoryLabel(directoryName);
-                                }
-                            }
-                        }
-                        if (new_datafile != null) {
-                            datafiles.add(new_datafile);
-                        } else {
-                            logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName());
-                        }
-                        finalFileInputStream.close();
-
-                    }
-                } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) {
-                    logger.severe("One of the unzipped shape files exceeded the size limit; giving up. " + femsx.getMessage());
-                    datafiles.clear();
-                }
-
-                // Delete the temp directory used for unzipping
-                // The try-catch is due to error encountered in using NFS for stocking file,
-                // cf. https://github.com/IQSS/dataverse/issues/5909
-                try {
-                    FileUtils.deleteDirectory(rezipFolder);
-                } catch (IOException ioex) {
-                    // do nothing - it's a tempo folder.
-                    logger.warning("Could not remove temp folder, error message : " + ioex.getMessage());
-                }
-
-                if (datafiles.size() > 0) {
-                    // remove the uploaded zip file:
-                    try {
-                        Files.delete(tempFile);
-                    } catch (IOException ioex) {
-                        // do nothing - it's just a temp file.
-                        logger.warning("Could not remove temp file " + tempFile.getFileName().toString());
-                    } catch (SecurityException se) {
-                        logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: "
-                                + se.getMessage());
-                    }
-                    return CreateDataFileResult.success(fileName, finalType, datafiles);
-                } else {
-                    logger.severe("No files added from directory of rezipped shapefiles");
-                }
-                return CreateDataFileResult.error(fileName, finalType);
-
-            } else if (finalType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE)) {
-                Optional<BagItFileHandler> bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
-                if (bagItFileHandler.isPresent()) {
-                    CreateDataFileResult result = bagItFileHandler.get().handleBagItPackage(systemConfig, version, fileName, tempFile.toFile());
-                    return result;
-                }
-            }
-        } else {
-            // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied
-            finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType;
-            String type = determineFileTypeByNameAndExtension(fileName);
-            if (!StringUtils.isBlank(type)) {
-                //Use rules for deciding when to trust browser supplied type
-                if (useRecognizedType(finalType, type)) {
-                    finalType = type;
-                }
-                logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType);
-            }
-        }
-        // Finally, if none of the special cases above were applicable (or 
-        // if we were unable to unpack an uploaded file, etc.), we'll just 
-        // create and return a single DataFile:
-        File newFile = null;
-        if (tempFile != null) {
-            newFile = tempFile.toFile();
-        }
-        
-
-        DataFile datafile = createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum);
-        File f = null;
-        if (tempFile != null) {
-            f = tempFile.toFile();
-        }
-        if (datafile != null && ((f != null) || (newStorageIdentifier != null))) {
-
-            if (warningMessage != null) {
-                createIngestFailureReport(datafile, warningMessage);
-                datafile.SetIngestProblem();
-            }
-            datafiles.add(datafile);
-
-            return CreateDataFileResult.success(fileName, finalType, datafiles);
-        }
-
-        return CreateDataFileResult.error(fileName, finalType);
-    }   // end createDataFiles
-    
 
 	public static boolean useRecognizedType(String suppliedContentType, String recognizedType) {
 		// is it any better than the type that was supplied to us,

From 1a22b11c65353f7c2bd0677b2f4bb2e134aebcb5 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 24 Feb 2023 17:02:39 -0500
Subject: [PATCH 033/922] Added info messages about the remaining storage
 quota, if enforced, for the user on the upload page. #9361

---
 .../iq/dataverse/EditDatafilesPage.java       | 20 +++++++++++++++----
 src/main/java/propertyFiles/Bundle.properties |  3 ++-
 src/main/webapp/editFilesFragment.xhtml       |  5 +++++
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index 928bf635ffa..420642f2fa5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -343,6 +343,11 @@ public Long getMaxFileUploadSizeInBytes() {
     public String getHumanMaxFileUploadSizeInBytes() {
         return FileSizeChecker.bytesToHumanReadable(this.maxFileUploadSizeInBytes);
     }
+
+    public boolean isUnlimitedUploadFileSize() {
+
+        return this.maxFileUploadSizeInBytes == null;
+    }
     
     public Long getMaxTotalUploadSizeInBytes() {
         return maxTotalUploadSizeInBytes;
@@ -351,10 +356,9 @@ public Long getMaxTotalUploadSizeInBytes() {
     public String getHumanMaxTotalUploadSizeInBytes() {
         return FileSizeChecker.bytesToHumanReadable(maxTotalUploadSizeInBytes);
     }
-
-    public boolean isUnlimitedUploadFileSize() {
-
-        return this.maxFileUploadSizeInBytes == null;
+    
+    public boolean isStorageQuotaEnforced() {
+        return maxTotalUploadSizeInBytes != null; 
     }
 
     public Long getMaxIngestSizeInBytes() {
@@ -524,6 +528,11 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo
         selectedFiles = selectedFileMetadatasList;
 
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
+        if (systemConfig.isStorageQuotasEnforced()) {
+            this.maxTotalUploadSizeInBytes = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset).getRemainingQuotaInBytes();
+        } else {
+            this.maxTotalUploadSizeInBytes = null; 
+        }
         this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
         this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
         this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit();
@@ -575,6 +584,9 @@ public String init() {
         
         clone = workingVersion.cloneDatasetVersion();
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
+        if (systemConfig.isStorageQuotasEnforced()) {
+            this.maxTotalUploadSizeInBytes = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset).getRemainingQuotaInBytes();
+        }
         this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
         this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
         this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit();        
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index c1fd4ebaf10..dd9b398b709 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1659,7 +1659,8 @@ file.select.tooltip=Select Files
 file.selectAllFiles=Select all {0} files in this dataset.
 file.dynamicCounter.filesPerPage=Files Per Page
 file.selectToAddBtn=Select Files to Add
-file.selectToAdd.tipLimit=File upload limit is {0} per file. 
+file.selectToAdd.tipLimit=File upload limit is {0} per file.
+file.selectToAdd.tipQuotaRemaining=Storage quota: {0} remaining. 
 file.selectToAdd.tipMaxNumFiles=Maximum of {0} {0, choice, 0#files|1#file|2#files} per upload.
 file.selectToAdd.tipTabularLimit=<a href="{0}/{1}/user/dataset-management.html#tabular-data-files" title="Tabular Data Files - Dataverse User Guide" target="_blank" rel="noopener">Tabular file ingest</a> is limited to {2}.
 file.selectToAdd.tipPerFileTabularLimit=Ingest is limited to the following file sizes based on their format: {0}.
diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml
index 99db5abd2dc..77f7aab2f76 100644
--- a/src/main/webapp/editFilesFragment.xhtml
+++ b/src/main/webapp/editFilesFragment.xhtml
@@ -80,6 +80,11 @@
                                                     rendered="#{!EditDatafilesPage.isUnlimitedUploadFileSize()}">
                                         <f:param value="#{EditDatafilesPage.getHumanMaxFileUploadSizeInBytes()}"/>
                                     </h:outputFormat>
+                                    
+                                    <h:outputFormat value=" #{bundle['file.selectToAdd.tipQuotaRemaining']}" escape="false" 
+                                                    rendered="#{EditDatafilesPage.isStorageQuotaEnforced()}">
+                                        <f:param value="#{EditDatafilesPage.getHumanMaxTotalUploadSizeInBytes()}"/>
+                                    </h:outputFormat>
                                     <h:outputFormat value=" #{bundle['file.selectToAdd.tipTabularLimit']}" escape="false"
                                                     rendered="#{EditDatafilesPage.maxIngestSizeInBytes != -1}">
                                         <f:param value="#{systemConfig.guidesBaseUrl}"/>

From 9bdcba600915d44117db9ae8ca046c1e32d07c3d Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Mon, 13 Mar 2023 11:54:39 -0400
Subject: [PATCH 034/922] Alternative Title

---
 conf/solr/8.11.1/schema.xml                   |  2 +-
 scripts/api/data/metadatablocks/citation.tsv  |  2 +-
 .../dataverse/export/ddi/DdiExportUtil.java   |  8 +++--
 .../export/openaire/OpenAireExportUtil.java   | 32 +++++++++++++++++--
 .../dataverse/export/dataset-all-defaults.txt |  4 +--
 .../dataset-create-new-all-ddi-fields.json    |  4 +--
 6 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml
index f11938621fc..5fa9c7f0616 100644
--- a/conf/solr/8.11.1/schema.xml
+++ b/conf/solr/8.11.1/schema.xml
@@ -250,7 +250,7 @@
     <!-- SCHEMA-FIELDS::BEGIN -->
     <field name="accessToSources" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="actionsToMinimizeLoss" type="text_en" multiValued="false" stored="true" indexed="true"/>
-    <field name="alternativeTitle" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="alternativeTitle" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="alternativeURL" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="astroFacility" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="astroInstrument" type="text_en" multiValued="true" stored="true" indexed="true"/>
diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv
index be32bb7134e..6b0f231c7b7 100644
--- a/scripts/api/data/metadatablocks/citation.tsv
+++ b/scripts/api/data/metadatablocks/citation.tsv
@@ -3,7 +3,7 @@
 #datasetField	name	title	description	watermark	 fieldType	displayOrder	displayFormat	advancedSearchField	allowControlledVocabulary	allowmultiples	facetable	displayoncreate	required	parent	metadatablock_id	termURI
 	title	Title	The main title of the Dataset		text	0		TRUE	FALSE	FALSE	FALSE	TRUE	TRUE		citation	http://purl.org/dc/terms/title
 	subtitle	Subtitle	A secondary title that amplifies or states certain limitations on the main title		text	1		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
-	alternativeTitle	Alternative Title	Either 1) a title commonly used to refer to the Dataset or 2) an abbreviation of the main title		text	2		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/alternative
+	alternativeTitle	Alternative Title	Either 1) a title commonly used to refer to the Dataset or 2) an abbreviation of the main title		text	2		FALSE	FALSE   TRUE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/alternative
 	alternativeURL	Alternative URL	Another URL where one can view or access the data in the Dataset, e.g. a project or personal webpage	https://	url	3	<a href="#VALUE" target="_blank">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	https://schema.org/distribution
 	otherId	Other Identifier	Another unique identifier for the Dataset (e.g. producer's or another repository's identifier)		none	4	:	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	otherIdAgency	Agency	The name of the agency that generated the other identifier		text	5	#VALUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	otherId	citation	
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index eb7632dd03c..eb53473d4d9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -202,8 +202,12 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
        
         writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage());
         writeFullElement(xmlw, "subTitl", dto2Primitive(version, DatasetFieldConstant.subTitle));
-        writeFullElement(xmlw, "altTitl", dto2Primitive(version, DatasetFieldConstant.alternativeTitle));
-        
+       
+        FieldDTO altField = dto2FieldDTO( version, DatasetFieldConstant.alternativeTitle, "citation"  );
+        if (altField != null) {
+            writeMultipleElement(xmlw, "altTitl", altField, datasetDto.getMetadataLanguage());
+        }
+
         xmlw.writeStartElement("IDNo");
         writeAttribute(xmlw, "agency", persistentAgency);
         
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
index bea3858a60e..34cb7a4e138 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
@@ -7,6 +7,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.logging.Logger;
+import java.util.List;
 
 import javax.json.JsonObject;
 import javax.xml.stream.XMLOutputFactory;
@@ -368,8 +369,8 @@ public static void writeTitlesElement(XMLStreamWriter xmlw, DatasetVersionDTO da
         String subtitle = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.subTitle);
         title_check = writeTitleElement(xmlw, "Subtitle", subtitle, title_check, language);
 
-        String alternativeTitle = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.alternativeTitle);
-        title_check = writeTitleElement(xmlw, "AlternativeTitle", alternativeTitle, title_check, language);
+        title_check = writeMultipleTitleElement(xmlw, "AlternativeTitle", datasetVersionDTO, "citation", title_check, language);
+
 
         writeEndTag(xmlw, title_check);
     }
@@ -404,6 +405,33 @@ private static boolean writeTitleElement(XMLStreamWriter xmlw, String titleType,
         }
         return title_check;
     }
+    
+    private static boolean writeMultipleTitleElement(XMLStreamWriter xmlw, String titleType, DatasetVersionDTO datasetVersionDTO, String metadataBlockName, boolean title_check, String language) throws XMLStreamException {
+        MetadataBlockDTO block = datasetVersionDTO.getMetadataBlocks().get(metadataBlockName);
+        if (block != null) {
+            logger.info("Block is not empty");
+            List<FieldDTO> fieldsBlock =  block.getFields();
+            if (fieldsBlock != null) {
+                for (FieldDTO fieldDTO : fieldsBlock) {
+                    logger.info(titleType + " " + fieldDTO.getTypeName());
+                    if (titleType.toLowerCase().equals(fieldDTO.getTypeName().toLowerCase())) {
+                        logger.info("Found Alt title");
+                        List<String> fields = fieldDTO.getMultiplePrimitive();
+                        for (String value : fields) {
+                            if (!writeTitleElement(xmlw, titleType, value, title_check, language))
+                                title_check = false;
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+
+        return title_check;
+    }
+
+
+
 
     /**
      * 5, PublicationYear (M)
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt
index a3f0dffc767..62f2cd37447 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt
+++ b/src/test/java/edu/harvard/iq/dataverse/export/dataset-all-defaults.txt
@@ -42,9 +42,9 @@
           },
           {
             "typeName": "alternativeTitle",
-            "multiple": false,
+            "multiple": true,
             "typeClass": "primitive",
-            "value": "Alternative Title"
+            "value": ["Alternative Title"]
           },
           {
             "typeName": "alternativeURL",
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
index 1b327c15496..96f058b1b02 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
+++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
@@ -51,9 +51,9 @@
           },
           {
             "typeName": "alternativeTitle",
-            "multiple": false,
+            "multiple": true,
             "typeClass": "primitive",
-            "value": "Alternative Title"
+            "value": ["Alternative Title"]
           },
           {
             "typeName": "otherId",

From 7b8281c7a0f98350533047dac1790603557e463b Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Mon, 13 Mar 2023 13:08:24 -0400
Subject: [PATCH 035/922] citation

---
 scripts/api/data/metadatablocks/citation.tsv | 509 +++++++++----------
 1 file changed, 254 insertions(+), 255 deletions(-)

diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv
index 6b0f231c7b7..20d858ddb4b 100644
--- a/scripts/api/data/metadatablocks/citation.tsv
+++ b/scripts/api/data/metadatablocks/citation.tsv
@@ -3,8 +3,8 @@
 #datasetField	name	title	description	watermark	 fieldType	displayOrder	displayFormat	advancedSearchField	allowControlledVocabulary	allowmultiples	facetable	displayoncreate	required	parent	metadatablock_id	termURI
 	title	Title	The main title of the Dataset		text	0		TRUE	FALSE	FALSE	FALSE	TRUE	TRUE		citation	http://purl.org/dc/terms/title
 	subtitle	Subtitle	A secondary title that amplifies or states certain limitations on the main title		text	1		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
-	alternativeTitle	Alternative Title	Either 1) a title commonly used to refer to the Dataset or 2) an abbreviation of the main title		text	2		FALSE	FALSE   TRUE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/alternative
-	alternativeURL	Alternative URL	Another URL where one can view or access the data in the Dataset, e.g. a project or personal webpage	https://	url	3	<a href="#VALUE" target="_blank">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	https://schema.org/distribution
+	alternativeTitle	Alternative Title	Either 1) a title commonly used to refer to the Dataset or 2) an abbreviation of the main title		text	2		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/alternative
+	alternativeURL	Alternative URL	Another URL where one can view or access the data in the Dataset, e.g. a project or personal webpage	https://	url	3	"<a href=""#VALUE"" target=""_blank"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	https://schema.org/distribution
 	otherId	Other Identifier	Another unique identifier for the Dataset (e.g. producer's or another repository's identifier)		none	4	:	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	otherIdAgency	Agency	The name of the agency that generated the other identifier		text	5	#VALUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	otherId	citation	
 	otherIdValue	Identifier	Another identifier uniquely identifies the Dataset		text	6	#VALUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	otherId	citation	
@@ -24,26 +24,26 @@
 	keyword	Keyword	A key term that describes an important aspect of the Dataset and information about any controlled vocabulary used		none	20		FALSE	FALSE	TRUE	FALSE	TRUE	FALSE		citation	
 	keywordValue	Term	A key term that describes important aspects of the Dataset		text	21	#VALUE	TRUE	FALSE	FALSE	TRUE	TRUE	FALSE	keyword	citation	
 	keywordVocabulary	Controlled Vocabulary Name	The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH)		text	22	(#VALUE)	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	keyword	citation	
-	keywordVocabularyURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	23	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	keyword	citation	
+	keywordVocabularyURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	23	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	keyword	citation	
 	topicClassification	Topic Classification	Indicates a broad, important topic or subject that the Dataset covers and information about any controlled vocabulary used		none	24		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	topicClassValue	Term	A topic or subject term		text	25	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	topicClassification	citation	
 	topicClassVocab	Controlled Vocabulary Name	The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH)		text	26	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	topicClassification	citation	
-	topicClassVocabURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	27	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	topicClassification	citation	
+	topicClassVocabURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	27	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	topicClassification	citation	
 	publication	Related Publication	The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab		none	28		FALSE	FALSE	TRUE	FALSE	TRUE	FALSE		citation	http://purl.org/dc/terms/isReferencedBy
 	publicationCitation	Citation	The full bibliographic citation for the related publication		textbox	29	#VALUE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	publication	citation	http://purl.org/dc/terms/bibliographicCitation
 	publicationIDType	Identifier Type	The type of identifier that uniquely identifies a related publication		text	30	#VALUE: 	TRUE	TRUE	FALSE	FALSE	TRUE	FALSE	publication	citation	http://purl.org/spar/datacite/ResourceIdentifierScheme
 	publicationIDNumber	Identifier	The identifier for a related publication		text	31	#VALUE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	publication	citation	http://purl.org/spar/datacite/ResourceIdentifier
-	publicationURL	URL	The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage	https://	url	32	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	publication	citation	https://schema.org/distribution
+	publicationURL	URL	The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage	https://	url	32	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	publication	citation	https://schema.org/distribution
 	notesText	Notes	Additional information about the Dataset		textbox	33		FALSE	FALSE	FALSE	FALSE	TRUE	FALSE		citation	
 	language	Language	A language that the Dataset's files is written in		text	34		TRUE	TRUE	TRUE	TRUE	FALSE	FALSE		citation	http://purl.org/dc/terms/language
 	producer	Producer	The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset		none	35		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	producerName	Name	The name of the entity, e.g. the person's name or the name of an organization	1) FamilyName, GivenName or 2) Organization	text	36	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	TRUE	producer	citation	
 	producerAffiliation	Affiliation	The name of the entity affiliated with the producer, e.g. an organization's name	Organization XYZ	text	37	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
 	producerAbbreviation	Abbreviated Name	The producer's abbreviated name (e.g. IQSS, ICPSR)		text	38	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
-	producerURL	URL	The URL of the producer's website	https://	url	39	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
-	producerLogoURL	Logo URL	The URL of the producer's logo	https://	url	40	<img src="#VALUE" alt="#NAME" class="metadata-logo"/><br/>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
+	producerURL	URL	The URL of the producer's website	https://	url	39	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
+	producerLogoURL	Logo URL	The URL of the producer's logo	https://	url	40	"<img src=""#VALUE"" alt=""#NAME"" class=""metadata-logo""/><br/>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
 	productionDate	Production Date	The date when the data were produced (not distributed, published, or archived)	YYYY-MM-DD	date	41		TRUE	FALSE	FALSE	TRUE	FALSE	FALSE		citation	
-	productionPlace	Production Location	The location where the data and any related materials were produced or collected		text	42		TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		citation	
+	productionPlace	Production Location	The location where the data and any related materials were produced or collected		text	42		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	contributor	Contributor	The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset		none	43	:	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/contributor
 	contributorType	Type	Indicates the type of contribution made to the dataset		text	44	#VALUE 	TRUE	TRUE	FALSE	TRUE	FALSE	FALSE	contributor	citation	
 	contributorName	Name	The name of the contributor, e.g. the person's name or the name of an organization	1) FamilyName, GivenName or 2) Organization	text	45	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	contributor	citation	
@@ -54,8 +54,8 @@
 	distributorName	Name	The name of the entity, e.g. the person's name or the name of an organization	1) FamilyName, GivenName or 2) Organization	text	50	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	distributor	citation	
 	distributorAffiliation	Affiliation	The name of the entity affiliated with the distributor, e.g. an organization's name	Organization XYZ	text	51	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
 	distributorAbbreviation	Abbreviated Name	The distributor's abbreviated name (e.g. IQSS, ICPSR)		text	52	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
-	distributorURL	URL	The URL of the distributor's webpage	https://	url	53	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
-	distributorLogoURL	Logo URL	The URL of the distributor's logo image, used to show the image on the Dataset's page	https://	url	54	<img src="#VALUE" alt="#NAME" class="metadata-logo"/><br/>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
+	distributorURL	URL	The URL of the distributor's webpage	https://	url	53	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
+	distributorLogoURL	Logo URL	The URL of the distributor's logo image, used to show the image on the Dataset's page	https://	url	54	"<img src=""#VALUE"" alt=""#NAME"" class=""metadata-logo""/><br/>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
 	distributionDate	Distribution Date	The date when the Dataset was made available for distribution/presentation	YYYY-MM-DD	date	55		TRUE	FALSE	FALSE	TRUE	FALSE	FALSE		citation	
 	depositor	Depositor	The entity, such as a person or organization, that deposited the Dataset in the repository	1) FamilyName, GivenName or 2) Organization	text	56		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	dateOfDeposit	Deposit Date	The date when the Dataset was deposited into the repository	YYYY-MM-DD	date	57		FALSE	FALSE	FALSE	TRUE	FALSE	FALSE		citation	http://purl.org/dc/terms/dateSubmitted
@@ -79,248 +79,247 @@
 	originOfSources	Origin of Historical Sources	For historical sources, the origin and any rules followed in establishing them as sources		textbox	75		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	characteristicOfSources	Characteristic of Sources	Characteristics not already noted elsewhere		textbox	76		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	accessToSources	Documentation and Access to Sources	1) Methods or procedures for accessing data sources and 2) any special permissions needed for access		textbox	77		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
-#controlledVocabulary	DatasetField	Value	identifier	displayOrder											
-	subject	Agricultural Sciences	D01	0											
-	subject	Arts and Humanities	D0	1											
-	subject	Astronomy and Astrophysics	D1	2											
-	subject	Business and Management	D2	3											
-	subject	Chemistry	D3	4
-	subject	Computer and Information Science	D7	5									
-	subject	Earth and Environmental Sciences	D4	6											
-	subject	Engineering	D5	7
-	subject	Law	D8	8	
-	subject	Mathematical Sciences	D9	9																	
-	subject	Medicine, Health and Life Sciences	D6	10											
-	subject	Physics	D10	11											
-	subject	Social Sciences	D11	12											
-	subject	Other	D12	13											
-	publicationIDType	ark		0											
-	publicationIDType	arXiv		1											
-	publicationIDType	bibcode		2								
-	publicationIDType	cstr		3											
-	publicationIDType	doi		4											
-	publicationIDType	ean13		5											
-	publicationIDType	eissn		6											
-	publicationIDType	handle		7											
-	publicationIDType	isbn		8											
-	publicationIDType	issn		9											
-	publicationIDType	istc		10											
-	publicationIDType	lissn		11											
-	publicationIDType	lsid		12											
-	publicationIDType	pmid		13											
-	publicationIDType	purl		14											
-	publicationIDType	upc		15											
-	publicationIDType	url		16	
-	publicationIDType	urn		17											
-	publicationIDType	DASH-NRS		18		
-	contributorType	Data Collector		0											
-	contributorType	Data Curator		1											
-	contributorType	Data Manager		2											
-	contributorType	Editor		3											
-	contributorType	Funder		4											
-	contributorType	Hosting Institution		5											
-	contributorType	Project Leader		6											
-	contributorType	Project Manager		7											
-	contributorType	Project Member		8											
-	contributorType	Related Person		9											
-	contributorType	Researcher		10											
-	contributorType	Research Group		11											
-	contributorType	Rights Holder		12											
-	contributorType	Sponsor		13											
-	contributorType	Supervisor		14											
-	contributorType	Work Package Leader		15											
-	contributorType	Other		16											
-	authorIdentifierScheme	ORCID		0											
-	authorIdentifierScheme	ISNI		1											
-	authorIdentifierScheme	LCNA		2											
-	authorIdentifierScheme	VIAF		3											
-	authorIdentifierScheme	GND		4											
-	authorIdentifierScheme	DAI		5
-	authorIdentifierScheme	ResearcherID		6
-	authorIdentifierScheme	ScopusID		7
-	language	Abkhaz		0
-	language	Afar		1	aar	aa
-	language	Afrikaans		2	afr	af
-	language	Akan		3	aka	ak
-	language	Albanian		4	sqi	alb	sq
-	language	Amharic		5	amh	am
-	language	Arabic		6	ara	ar
-	language	Aragonese		7	arg	an
-	language	Armenian		8	hye	arm	hy
-	language	Assamese		9	asm	as
-	language	Avaric		10	ava	av
-	language	Avestan		11	ave	ae
-	language	Aymara		12	aym	ay
-	language	Azerbaijani		13	aze	az
-	language	Bambara		14	bam	bm
-	language	Bashkir		15	bak	ba
-	language	Basque		16	eus	baq	eu
-	language	Belarusian		17	bel	be
-	language	Bengali, Bangla		18	ben	bn
-	language	Bihari		19	bih	bh
-	language	Bislama		20	bis	bi
-	language	Bosnian		21	bos	bs
-	language	Breton		22	bre	br
-	language	Bulgarian		23	bul	bg
-	language	Burmese		24	mya	bur	my
-	language	Catalan,Valencian		25	cat ca
-	language	Chamorro		26	cha	ch
-	language	Chechen		27	che	ce
-	language	Chichewa, Chewa, Nyanja		28	nya	ny
-	language	Chinese		29	zho	chi	zh
-	language	Chuvash		30	chv	cv
-	language	Cornish		31	cor	kw
-	language	Corsican		32	cos	co
-	language	Cree		33	cre	cr
-	language	Croatian		34	hrv	src	hr
-	language	Czech		35	ces	cze	cs
-	language	Danish		36	dan	da
-	language	Divehi, Dhivehi, Maldivian		37	div	dv
-	language	Dutch		38	nld	dut	nl
-	language	Dzongkha		39	dzo	dz
-	language	English		40	eng	en
-	language	Esperanto		41	epo	eo
-	language	Estonian		42	est	et
-	language	Ewe		43	ewe	ee
-	language	Faroese		44	fao	fo
-	language	Fijian		45	fij	fj
-	language	Finnish		46	fin	fi
-	language	French		47	fra	fre	fr
-	language	Fula, Fulah, Pulaar, Pular		48	ful	ff
-	language	Galician		49	glg	gl
-	language	Georgian		50	kat	geo	ka
-	language	German		51	deu	ger	de
-	language	Greek (modern)		52	gre	ell	el
-	language	Guaraní		53	grn	gn
-	language	Gujarati		54	guj	gu
-	language	Haitian, Haitian Creole		55	hat ht
-	language	Hausa		56	hau	ha
-	language	Hebrew (modern)		57	heb	he
-	language	Herero		58	her	hz
-	language	Hindi		59	hin	hi
-	language	Hiri Motu		60	hmo	ho
-	language	Hungarian		61	hun	hu
-	language	Interlingua		62	ina	ia
-	language	Indonesian		63	ind	id
-	language	Interlingue		64	ile	ie
-	language	Irish		65	gle	ga
-	language	Igbo		66	ibo	ig
-	language	Inupiaq		67	ipk	ik
-	language	Ido		68	ido	io
-	language	Icelandic		69	isl	ice	is
-	language	Italian		70	ita	it
-	language	Inuktitut		71	iku	iu
-	language	Japanese		72	jpn	ja
-	language	Javanese		73	jav	jv
-	language	Kalaallisut, Greenlandic		74	kal	kl
-	language	Kannada		75	kan	kn
-	language	Kanuri		76	kau	kr
-	language	Kashmiri		77	kas	ks
-	language	Kazakh		78	kaz	kk
-	language	Khmer		79	khm	km
-	language	Kikuyu, Gikuyu		80	kik	ki
-	language	Kinyarwanda		81	kin	rw
-	language	Kyrgyz		82											
-	language	Komi		83	kom	kv
-	language	Kongo		84	kon	kg
-	language	Korean		85	kor	ko
-	language	Kurdish		86	kur	ku
-	language	Kwanyama, Kuanyama		87	kua	kj
-	language	Latin		88	lat	la
-	language	Luxembourgish, Letzeburgesch		89	ltz	lb
-	language	Ganda		90	lug	lg
-	language	Limburgish, Limburgan, Limburger		91	lim	li
-	language	Lingala		92	lin	ln
-	language	Lao		93	lao	lo
-	language	Lithuanian		94	lit	lt
-	language	Luba-Katanga		95	lub	lu
-	language	Latvian		96	lav	lv
-	language	Manx		97	glv	gv
-	language	Macedonian		98	mkd	mac	mk
-	language	Malagasy		99	mlg	mg
-	language	Malay		100	may	msa	ms
-	language	Malayalam		101	mal	ml
-	language	Maltese		102	mlt	mt
-	language	M�ori		103	mao	mri	mi
-	language	Marathi (Mar�ṭhī)		104	mar	mr
-	language	Marshallese		105	mah	mh
-	language	Mixtepec Mixtec		106	mix
-	language	Mongolian		107	mon	mn
-	language	Nauru		108	nau	na
-	language	Navajo, Navaho		109	nav	nv
-	language	Northern Ndebele		110	nde	nd
-	language	Nepali		111	nep	ne
-	language	Ndonga		112	ndo	ng
-	language	Norwegian Bokmål		113	nob	nb
-	language	Norwegian Nynorsk		114	nno	nn
-	language	Norwegian		115	nor	no
-	language	Nuosu		116											
-	language	Southern Ndebele		117	nbl	nr
-	language	Occitan		118	oci	oc
-	language	Ojibwe, Ojibwa		119	oji	oj
-	language	Old Church Slavonic,Church Slavonic,Old Bulgarian		120	chu	cu
-	language	Oromo		121	orm	om
-	language	Oriya		122	ori	or
-	language	Ossetian, Ossetic		123	oss	os
-	language	Panjabi, Punjabi		124	pan	pa
-	language	P�li		125	pli	pi
-	language	Persian (Farsi)		126	per	fas	fa
-	language	Polish		127	pol	pl
-	language	Pashto, Pushto		128	pus	ps
-	language	Portuguese		129	por	pt
-	language	Quechua		130	que	qu
-	language	Romansh		131	roh	rm
-	language	Kirundi		132	run	rn
-	language	Romanian		133	ron	rum	ro
-	language	Russian		134	rus	ru
-	language	Sanskrit (Sa�skṛta)		135	san	sa
-	language	Sardinian		136	srd	sc
-	language	Sindhi		137	snd	sd
-	language	Northern Sami		138	sme	se
-	language	Samoan		139	smo	sm
-	language	Sango		140	sag	sg
-	language	Serbian		141	srp	scc	sr
-	language	Scottish Gaelic, Gaelic		142	gla	gd
-	language	Shona		143	sna	sn
-	language	Sinhala, Sinhalese		144	sin	si
-	language	Slovak		145	slk	slo	sk
-	language	Slovene		146	slv	sl
-	language	Somali		147	som	so
-	language	Southern Sotho		148	sot	st
-	language	Spanish, Castilian		149	spa	es
-	language	Sundanese		150	sun	su
-	language	Swahili		151	swa	sw
-	language	Swati		152	ssw	ss
-	language	Swedish		153	swe	sv
-	language	Tamil		154	tam	ta
-	language	Telugu		155	tel	te
-	language	Tajik		156	tgk	tg
-	language	Thai		157	tha	th
-	language	Tigrinya		158	tir	ti
-	language	Tibetan Standard, Tibetan, Central		159	tib	bod	bo
-	language	Turkmen		160	tuk	tk
-	language	Tagalog		161	tgl	tl
-	language	Tswana		162	tsn	tn
-	language	Tonga (Tonga Islands)		163	ton	to
-	language	Turkish		164	tur	tr
-	language	Tsonga		165	tso	ts
-	language	Tatar		166	tat	tt
-	language	Twi		167	twi	tw
-	language	Tahitian		168	tah	ty
-	language	Uyghur, Uighur		169	uig	ug
-	language	Ukrainian		170	ukr	uk
-	language	Urdu		171	urd	ur
-	language	Uzbek		172	uzb	uz
-	language	Venda		173	ven	ve
-	language	Vietnamese		174	vie	vi
-	language	Volapük		175	vol	vo
-	language	Walloon		176	wln	wa
-	language	Welsh		177	cym	wel	cy
-	language	Wolof		178	wol	wo
-	language	Western Frisian		179	fry	fy
-	language	Xhosa		180	xho	xh
-	language	Yiddish		181	yid	yi
-	language	Yoruba		182	yor	yo
-	language	Zhuang, Chuang		183	zha	za
-	language	Zulu		184	zul	zu
-	language	Not applicable		185											
+#controlledVocabulary	DatasetField	Value	identifier	displayOrder												
+	subject	Agricultural Sciences	D01	0												
+	subject	Arts and Humanities	D0	1												
+	subject	Astronomy and Astrophysics	D1	2												
+	subject	Business and Management	D2	3												
+	subject	Chemistry	D3	4												
+	subject	Computer and Information Science	D7	5												
+	subject	Earth and Environmental Sciences	D4	6												
+	subject	Engineering	D5	7												
+	subject	Law	D8	8												
+	subject	Mathematical Sciences	D9	9												
+	subject	Medicine, Health and Life Sciences	D6	10												
+	subject	Physics	D10	11												
+	subject	Social Sciences	D11	12												
+	subject	Other	D12	13												
+	publicationIDType	ark		0												
+	publicationIDType	arXiv		1												
+	publicationIDType	bibcode		2												
+	publicationIDType	doi		3												
+	publicationIDType	ean13		4												
+	publicationIDType	eissn		5												
+	publicationIDType	handle		6												
+	publicationIDType	isbn		7												
+	publicationIDType	issn		8												
+	publicationIDType	istc		9												
+	publicationIDType	lissn		10												
+	publicationIDType	lsid		11												
+	publicationIDType	pmid		12												
+	publicationIDType	purl		13												
+	publicationIDType	upc		14												
+	publicationIDType	url		15												
+	publicationIDType	urn		16												
+	publicationIDType	DASH-NRS		17												
+	contributorType	Data Collector		0												
+	contributorType	Data Curator		1												
+	contributorType	Data Manager		2												
+	contributorType	Editor		3												
+	contributorType	Funder		4												
+	contributorType	Hosting Institution		5												
+	contributorType	Project Leader		6												
+	contributorType	Project Manager		7												
+	contributorType	Project Member		8												
+	contributorType	Related Person		9												
+	contributorType	Researcher		10												
+	contributorType	Research Group		11												
+	contributorType	Rights Holder		12												
+	contributorType	Sponsor		13												
+	contributorType	Supervisor		14												
+	contributorType	Work Package Leader		15												
+	contributorType	Other		16												
+	authorIdentifierScheme	ORCID		0												
+	authorIdentifierScheme	ISNI		1												
+	authorIdentifierScheme	LCNA		2												
+	authorIdentifierScheme	VIAF		3												
+	authorIdentifierScheme	GND		4												
+	authorIdentifierScheme	DAI		5												
+	authorIdentifierScheme	ResearcherID		6												
+	authorIdentifierScheme	ScopusID		7												
+	language	Abkhaz		0												
+	language	Afar		1	aar	aa										
+	language	Afrikaans		2	afr	af										
+	language	Akan		3	aka	ak										
+	language	Albanian		4	sqi	alb	sq									
+	language	Amharic		5	amh	am										
+	language	Arabic		6	ara	ar										
+	language	Aragonese		7	arg	an										
+	language	Armenian		8	hye	arm	hy									
+	language	Assamese		9	asm	as										
+	language	Avaric		10	ava	av										
+	language	Avestan		11	ave	ae										
+	language	Aymara		12	aym	ay										
+	language	Azerbaijani		13	aze	az										
+	language	Bambara		14	bam	bm										
+	language	Bashkir		15	bak	ba										
+	language	Basque		16	eus	baq	eu									
+	language	Belarusian		17	bel	be										
+	language	Bengali, Bangla		18	ben	bn										
+	language	Bihari		19	bih	bh										
+	language	Bislama		20	bis	bi										
+	language	Bosnian		21	bos	bs										
+	language	Breton		22	bre	br										
+	language	Bulgarian		23	bul	bg										
+	language	Burmese		24	mya	bur	my									
+	language	Catalan,Valencian		25	cat ca											
+	language	Chamorro		26	cha	ch										
+	language	Chechen		27	che	ce										
+	language	Chichewa, Chewa, Nyanja		28	nya	ny										
+	language	Chinese		29	zho	chi	zh									
+	language	Chuvash		30	chv	cv										
+	language	Cornish		31	cor	kw										
+	language	Corsican		32	cos	co										
+	language	Cree		33	cre	cr										
+	language	Croatian		34	hrv	src	hr									
+	language	Czech		35	ces	cze	cs									
+	language	Danish		36	dan	da										
+	language	Divehi, Dhivehi, Maldivian		37	div	dv										
+	language	Dutch		38	nld	dut	nl									
+	language	Dzongkha		39	dzo	dz										
+	language	English		40	eng	en										
+	language	Esperanto		41	epo	eo										
+	language	Estonian		42	est	et										
+	language	Ewe		43	ewe	ee										
+	language	Faroese		44	fao	fo										
+	language	Fijian		45	fij	fj										
+	language	Finnish		46	fin	fi										
+	language	French		47	fra	fre	fr									
+	language	Fula, Fulah, Pulaar, Pular		48	ful	ff										
+	language	Galician		49	glg	gl										
+	language	Georgian		50	kat	geo	ka									
+	language	German		51	deu	ger	de									
+	language	Greek (modern)		52	gre	ell	el									
+	language	Guaraní		53	grn	gn										
+	language	Gujarati		54	guj	gu										
+	language	Haitian, Haitian Creole		55	hat ht											
+	language	Hausa		56	hau	ha										
+	language	Hebrew (modern)		57	heb	he										
+	language	Herero		58	her	hz										
+	language	Hindi		59	hin	hi										
+	language	Hiri Motu		60	hmo	ho										
+	language	Hungarian		61	hun	hu										
+	language	Interlingua		62	ina	ia										
+	language	Indonesian		63	ind	id										
+	language	Interlingue		64	ile	ie										
+	language	Irish		65	gle	ga										
+	language	Igbo		66	ibo	ig										
+	language	Inupiaq		67	ipk	ik										
+	language	Ido		68	ido	io										
+	language	Icelandic		69	isl	ice	is									
+	language	Italian		70	ita	it										
+	language	Inuktitut		71	iku	iu										
+	language	Japanese		72	jpn	ja										
+	language	Javanese		73	jav	jv										
+	language	Kalaallisut, Greenlandic		74	kal	kl										
+	language	Kannada		75	kan	kn										
+	language	Kanuri		76	kau	kr										
+	language	Kashmiri		77	kas	ks										
+	language	Kazakh		78	kaz	kk										
+	language	Khmer		79	khm	km										
+	language	Kikuyu, Gikuyu		80	kik	ki										
+	language	Kinyarwanda		81	kin	rw										
+	language	Kyrgyz		82												
+	language	Komi		83	kom	kv										
+	language	Kongo		84	kon	kg										
+	language	Korean		85	kor	ko										
+	language	Kurdish		86	kur	ku										
+	language	Kwanyama, Kuanyama		87	kua	kj										
+	language	Latin		88	lat	la										
+	language	Luxembourgish, Letzeburgesch		89	ltz	lb										
+	language	Ganda		90	lug	lg										
+	language	Limburgish, Limburgan, Limburger		91	lim	li										
+	language	Lingala		92	lin	ln										
+	language	Lao		93	lao	lo										
+	language	Lithuanian		94	lit	lt										
+	language	Luba-Katanga		95	lub	lu										
+	language	Latvian		96	lav	lv										
+	language	Manx		97	glv	gv										
+	language	Macedonian		98	mkd	mac	mk									
+	language	Malagasy		99	mlg	mg										
+	language	Malay		100	may	msa	ms									
+	language	Malayalam		101	mal	ml										
+	language	Maltese		102	mlt	mt										
+	language	M�ori		103	mao	mri	mi									
+	language	Marathi (Mar�ṭhī)		104	mar	mr										
+	language	Marshallese		105	mah	mh										
+	language	Mixtepec Mixtec		106	mix											
+	language	Mongolian		107	mon	mn										
+	language	Nauru		108	nau	na										
+	language	Navajo, Navaho		109	nav	nv										
+	language	Northern Ndebele		110	nde	nd										
+	language	Nepali		111	nep	ne										
+	language	Ndonga		112	ndo	ng										
+	language	Norwegian Bokmål		113	nob	nb										
+	language	Norwegian Nynorsk		114	nno	nn										
+	language	Norwegian		115	nor	no										
+	language	Nuosu		116												
+	language	Southern Ndebele		117	nbl	nr										
+	language	Occitan		118	oci	oc										
+	language	Ojibwe, Ojibwa		119	oji	oj										
+	language	Old Church Slavonic,Church Slavonic,Old Bulgarian		120	chu	cu										
+	language	Oromo		121	orm	om										
+	language	Oriya		122	ori	or										
+	language	Ossetian, Ossetic		123	oss	os										
+	language	Panjabi, Punjabi		124	pan	pa										
+	language	P�li		125	pli	pi										
+	language	Persian (Farsi)		126	per	fas	fa									
+	language	Polish		127	pol	pl										
+	language	Pashto, Pushto		128	pus	ps										
+	language	Portuguese		129	por	pt										
+	language	Quechua		130	que	qu										
+	language	Romansh		131	roh	rm										
+	language	Kirundi		132	run	rn										
+	language	Romanian		133	ron	rum	ro									
+	language	Russian		134	rus	ru										
+	language	Sanskrit (Sa�skṛta)		135	san	sa										
+	language	Sardinian		136	srd	sc										
+	language	Sindhi		137	snd	sd										
+	language	Northern Sami		138	sme	se										
+	language	Samoan		139	smo	sm										
+	language	Sango		140	sag	sg										
+	language	Serbian		141	srp	scc	sr									
+	language	Scottish Gaelic, Gaelic		142	gla	gd										
+	language	Shona		143	sna	sn										
+	language	Sinhala, Sinhalese		144	sin	si										
+	language	Slovak		145	slk	slo	sk									
+	language	Slovene		146	slv	sl										
+	language	Somali		147	som	so										
+	language	Southern Sotho		148	sot	st										
+	language	Spanish, Castilian		149	spa	es										
+	language	Sundanese		150	sun	su										
+	language	Swahili		151	swa	sw										
+	language	Swati		152	ssw	ss										
+	language	Swedish		153	swe	sv										
+	language	Tamil		154	tam	ta										
+	language	Telugu		155	tel	te										
+	language	Tajik		156	tgk	tg										
+	language	Thai		157	tha	th										
+	language	Tigrinya		158	tir	ti										
+	language	Tibetan Standard, Tibetan, Central		159	tib	bod	bo									
+	language	Turkmen		160	tuk	tk										
+	language	Tagalog		161	tgl	tl										
+	language	Tswana		162	tsn	tn										
+	language	Tonga (Tonga Islands)		163	ton	to										
+	language	Turkish		164	tur	tr										
+	language	Tsonga		165	tso	ts										
+	language	Tatar		166	tat	tt										
+	language	Twi		167	twi	tw										
+	language	Tahitian		168	tah	ty										
+	language	Uyghur, Uighur		169	uig	ug										
+	language	Ukrainian		170	ukr	uk										
+	language	Urdu		171	urd	ur										
+	language	Uzbek		172	uzb	uz										
+	language	Venda		173	ven	ve										
+	language	Vietnamese		174	vie	vi										
+	language	Volapük		175	vol	vo										
+	language	Walloon		176	wln	wa										
+	language	Welsh		177	cym	wel	cy									
+	language	Wolof		178	wol	wo										
+	language	Western Frisian		179	fry	fy										
+	language	Xhosa		180	xho	xh										
+	language	Yiddish		181	yid	yi										
+	language	Yoruba		182	yor	yo										
+	language	Zhuang, Chuang		183	zha	za										
+	language	Zulu		184	zul	zu										
+	language	Not applicable		185												

From e5a356a5132cb7296ca1878f0120f0a52faebbdb Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Mon, 13 Mar 2023 13:35:42 -0400
Subject: [PATCH 036/922] release notes

---
 doc/release-notes/9428-alternative-title.md | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 doc/release-notes/9428-alternative-title.md

diff --git a/doc/release-notes/9428-alternative-title.md b/doc/release-notes/9428-alternative-title.md
new file mode 100644
index 00000000000..d6eaa680612
--- /dev/null
+++ b/doc/release-notes/9428-alternative-title.md
@@ -0,0 +1,6 @@
+Alternative Title is made repeatable. 
+- One will need to update database with updated citation block.
+`curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
+- One will also need to update solr schema:
+Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml` 
+Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"` 

From 5d81fc7042e81b152cc89773a3e67afd47c17e5a Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Tue, 14 Mar 2023 16:15:48 -0400
Subject: [PATCH 037/922] import

---
 .../iq/dataverse/api/imports/ImportDDIServiceBean.java     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index d9433832309..f7f0e30ea6e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -1396,6 +1396,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
    private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException, ImportException {
        MetadataBlockDTO citation = datasetDTO.getDatasetVersion().getMetadataBlocks().get("citation");
        List<HashSet<FieldDTO>> otherIds = new ArrayList<>();
+       List<String> altTitles = new ArrayList<>();
        
        for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
             if (event == XMLStreamConstants.START_ELEMENT) {
@@ -1406,8 +1407,7 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws
                   FieldDTO field = FieldDTO.createPrimitiveFieldDTO("subtitle", parseText(xmlr));
                    citation.getFields().add(field);
                 } else if (xmlr.getLocalName().equals("altTitl")) {
-                  FieldDTO field = FieldDTO.createPrimitiveFieldDTO("alternativeTitle", parseText(xmlr));
-                   citation.getFields().add(field);
+                    altTitles.add(parseText(xmlr));
                 } else if (xmlr.getLocalName().equals("IDNo")) {
                     if ( AGENCY_HANDLE.equals( xmlr.getAttributeValue(null, "agency") ) || AGENCY_DOI.equals( xmlr.getAttributeValue(null, "agency") ) ) {
                         importGenericService.reassignIdentifierAsGlobalId(parseText(xmlr), datasetDTO);
@@ -1435,6 +1435,9 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws
                     if (otherIds.size()>0) {
                         citation.addField(FieldDTO.createMultipleCompoundFieldDTO("otherId", otherIds));
                     }
+                    if (altTitles.size()>0) {
+                        citation.addField(FieldDTO.createMultiplePrimitiveFieldDTO("alternativeTitle", altTitles));
+                    }
                     return;
                 }
             }

From 991c5f9faf5a378c0c9e21848788d24cb918fbbc Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Wed, 15 Mar 2023 17:20:01 -0400
Subject: [PATCH 038/922] prodPlac

---
 scripts/api/data/metadatablocks/citation.tsv  | 507 +++++++++---------
 .../api/imports/ImportDDIServiceBean.java     |   6 +-
 .../dataverse/export/ddi/DdiExportUtil.java   |   9 +-
 .../edu/harvard/iq/dataverse/api/AdminIT.java |   2 +-
 .../dataset-create-new-all-ddi-fields.json    |   4 +-
 5 files changed, 268 insertions(+), 260 deletions(-)

diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv
index 20d858ddb4b..3fbc38438b8 100644
--- a/scripts/api/data/metadatablocks/citation.tsv
+++ b/scripts/api/data/metadatablocks/citation.tsv
@@ -4,7 +4,7 @@
 	title	Title	The main title of the Dataset		text	0		TRUE	FALSE	FALSE	FALSE	TRUE	TRUE		citation	http://purl.org/dc/terms/title
 	subtitle	Subtitle	A secondary title that amplifies or states certain limitations on the main title		text	1		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	alternativeTitle	Alternative Title	Either 1) a title commonly used to refer to the Dataset or 2) an abbreviation of the main title		text	2		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/alternative
-	alternativeURL	Alternative URL	Another URL where one can view or access the data in the Dataset, e.g. a project or personal webpage	https://	url	3	"<a href=""#VALUE"" target=""_blank"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	https://schema.org/distribution
+	alternativeURL	Alternative URL	Another URL where one can view or access the data in the Dataset, e.g. a project or personal webpage	https://	url	3	<a href="#VALUE" target="_blank">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	https://schema.org/distribution
 	otherId	Other Identifier	Another unique identifier for the Dataset (e.g. producer's or another repository's identifier)		none	4	:	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	otherIdAgency	Agency	The name of the agency that generated the other identifier		text	5	#VALUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	otherId	citation	
 	otherIdValue	Identifier	Another identifier uniquely identifies the Dataset		text	6	#VALUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	otherId	citation	
@@ -24,26 +24,26 @@
 	keyword	Keyword	A key term that describes an important aspect of the Dataset and information about any controlled vocabulary used		none	20		FALSE	FALSE	TRUE	FALSE	TRUE	FALSE		citation	
 	keywordValue	Term	A key term that describes important aspects of the Dataset		text	21	#VALUE	TRUE	FALSE	FALSE	TRUE	TRUE	FALSE	keyword	citation	
 	keywordVocabulary	Controlled Vocabulary Name	The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH)		text	22	(#VALUE)	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	keyword	citation	
-	keywordVocabularyURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	23	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	keyword	citation	
+	keywordVocabularyURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	23	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	keyword	citation	
 	topicClassification	Topic Classification	Indicates a broad, important topic or subject that the Dataset covers and information about any controlled vocabulary used		none	24		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	topicClassValue	Term	A topic or subject term		text	25	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	topicClassification	citation	
 	topicClassVocab	Controlled Vocabulary Name	The controlled vocabulary used for the keyword term (e.g. LCSH, MeSH)		text	26	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	topicClassification	citation	
-	topicClassVocabURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	27	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	topicClassification	citation	
+	topicClassVocabURI	Controlled Vocabulary URL	The URL where one can access information about the term's controlled vocabulary	https://	url	27	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	topicClassification	citation	
 	publication	Related Publication	The article or report that uses the data in the Dataset. The full list of related publications will be displayed on the metadata tab		none	28		FALSE	FALSE	TRUE	FALSE	TRUE	FALSE		citation	http://purl.org/dc/terms/isReferencedBy
 	publicationCitation	Citation	The full bibliographic citation for the related publication		textbox	29	#VALUE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	publication	citation	http://purl.org/dc/terms/bibliographicCitation
 	publicationIDType	Identifier Type	The type of identifier that uniquely identifies a related publication		text	30	#VALUE: 	TRUE	TRUE	FALSE	FALSE	TRUE	FALSE	publication	citation	http://purl.org/spar/datacite/ResourceIdentifierScheme
 	publicationIDNumber	Identifier	The identifier for a related publication		text	31	#VALUE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	publication	citation	http://purl.org/spar/datacite/ResourceIdentifier
-	publicationURL	URL	The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage	https://	url	32	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	publication	citation	https://schema.org/distribution
+	publicationURL	URL	The URL form of the identifier entered in the Identifier field, e.g. the DOI URL if a DOI was entered in the Identifier field. Used to display what was entered in the ID Type and ID Number fields as a link. If what was entered in the Identifier field has no URL form, the URL of the publication webpage is used, e.g. a journal article webpage	https://	url	32	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	publication	citation	https://schema.org/distribution
 	notesText	Notes	Additional information about the Dataset		textbox	33		FALSE	FALSE	FALSE	FALSE	TRUE	FALSE		citation	
 	language	Language	A language that the Dataset's files is written in		text	34		TRUE	TRUE	TRUE	TRUE	FALSE	FALSE		citation	http://purl.org/dc/terms/language
 	producer	Producer	The entity, such a person or organization, managing the finances or other administrative processes involved in the creation of the Dataset		none	35		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	
 	producerName	Name	The name of the entity, e.g. the person's name or the name of an organization	1) FamilyName, GivenName or 2) Organization	text	36	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	TRUE	producer	citation	
 	producerAffiliation	Affiliation	The name of the entity affiliated with the producer, e.g. an organization's name	Organization XYZ	text	37	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
 	producerAbbreviation	Abbreviated Name	The producer's abbreviated name (e.g. IQSS, ICPSR)		text	38	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
-	producerURL	URL	The URL of the producer's website	https://	url	39	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
-	producerLogoURL	Logo URL	The URL of the producer's logo	https://	url	40	"<img src=""#VALUE"" alt=""#NAME"" class=""metadata-logo""/><br/>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
+	producerURL	URL	The URL of the producer's website	https://	url	39	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
+	producerLogoURL	Logo URL	The URL of the producer's logo	https://	url	40	<img src="#VALUE" alt="#NAME" class="metadata-logo"/><br/>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	producer	citation	
 	productionDate	Production Date	The date when the data were produced (not distributed, published, or archived)	YYYY-MM-DD	date	41		TRUE	FALSE	FALSE	TRUE	FALSE	FALSE		citation	
-	productionPlace	Production Location	The location where the data and any related materials were produced or collected		text	42		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
+	productionPlace	Production Location	The location where the data and any related materials were produced or collected		text	42		TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		citation	
 	contributor	Contributor	The entity, such as a person or organization, responsible for collecting, managing, or otherwise contributing to the development of the Dataset		none	43	:	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		citation	http://purl.org/dc/terms/contributor
 	contributorType	Type	Indicates the type of contribution made to the dataset		text	44	#VALUE 	TRUE	TRUE	FALSE	TRUE	FALSE	FALSE	contributor	citation	
 	contributorName	Name	The name of the contributor, e.g. the person's name or the name of an organization	1) FamilyName, GivenName or 2) Organization	text	45	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	contributor	citation	
@@ -54,8 +54,8 @@
 	distributorName	Name	The name of the entity, e.g. the person's name or the name of an organization	1) FamilyName, GivenName or 2) Organization	text	50	#VALUE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	distributor	citation	
 	distributorAffiliation	Affiliation	The name of the entity affiliated with the distributor, e.g. an organization's name	Organization XYZ	text	51	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
 	distributorAbbreviation	Abbreviated Name	The distributor's abbreviated name (e.g. IQSS, ICPSR)		text	52	(#VALUE)	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
-	distributorURL	URL	The URL of the distributor's webpage	https://	url	53	"<a href=""#VALUE"" target=""_blank"" rel=""noopener"">#VALUE</a>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
-	distributorLogoURL	Logo URL	The URL of the distributor's logo image, used to show the image on the Dataset's page	https://	url	54	"<img src=""#VALUE"" alt=""#NAME"" class=""metadata-logo""/><br/>"	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
+	distributorURL	URL	The URL of the distributor's webpage	https://	url	53	<a href="#VALUE" target="_blank" rel="noopener">#VALUE</a>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
+	distributorLogoURL	Logo URL	The URL of the distributor's logo image, used to show the image on the Dataset's page	https://	url	54	<img src="#VALUE" alt="#NAME" class="metadata-logo"/><br/>	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	distributor	citation	
 	distributionDate	Distribution Date	The date when the Dataset was made available for distribution/presentation	YYYY-MM-DD	date	55		TRUE	FALSE	FALSE	TRUE	FALSE	FALSE		citation	
 	depositor	Depositor	The entity, such as a person or organization, that deposited the Dataset in the repository	1) FamilyName, GivenName or 2) Organization	text	56		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	dateOfDeposit	Deposit Date	The date when the Dataset was deposited into the repository	YYYY-MM-DD	date	57		FALSE	FALSE	FALSE	TRUE	FALSE	FALSE		citation	http://purl.org/dc/terms/dateSubmitted
@@ -79,247 +79,248 @@
 	originOfSources	Origin of Historical Sources	For historical sources, the origin and any rules followed in establishing them as sources		textbox	75		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	characteristicOfSources	Characteristic of Sources	Characteristics not already noted elsewhere		textbox	76		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
 	accessToSources	Documentation and Access to Sources	1) Methods or procedures for accessing data sources and 2) any special permissions needed for access		textbox	77		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE		citation	
-#controlledVocabulary	DatasetField	Value	identifier	displayOrder												
-	subject	Agricultural Sciences	D01	0												
-	subject	Arts and Humanities	D0	1												
-	subject	Astronomy and Astrophysics	D1	2												
-	subject	Business and Management	D2	3												
-	subject	Chemistry	D3	4												
-	subject	Computer and Information Science	D7	5												
-	subject	Earth and Environmental Sciences	D4	6												
-	subject	Engineering	D5	7												
-	subject	Law	D8	8												
-	subject	Mathematical Sciences	D9	9												
-	subject	Medicine, Health and Life Sciences	D6	10												
-	subject	Physics	D10	11												
-	subject	Social Sciences	D11	12												
-	subject	Other	D12	13												
-	publicationIDType	ark		0												
-	publicationIDType	arXiv		1												
-	publicationIDType	bibcode		2												
-	publicationIDType	doi		3												
-	publicationIDType	ean13		4												
-	publicationIDType	eissn		5												
-	publicationIDType	handle		6												
-	publicationIDType	isbn		7												
-	publicationIDType	issn		8												
-	publicationIDType	istc		9												
-	publicationIDType	lissn		10												
-	publicationIDType	lsid		11												
-	publicationIDType	pmid		12												
-	publicationIDType	purl		13												
-	publicationIDType	upc		14												
-	publicationIDType	url		15												
-	publicationIDType	urn		16												
-	publicationIDType	DASH-NRS		17												
-	contributorType	Data Collector		0												
-	contributorType	Data Curator		1												
-	contributorType	Data Manager		2												
-	contributorType	Editor		3												
-	contributorType	Funder		4												
-	contributorType	Hosting Institution		5												
-	contributorType	Project Leader		6												
-	contributorType	Project Manager		7												
-	contributorType	Project Member		8												
-	contributorType	Related Person		9												
-	contributorType	Researcher		10												
-	contributorType	Research Group		11												
-	contributorType	Rights Holder		12												
-	contributorType	Sponsor		13												
-	contributorType	Supervisor		14												
-	contributorType	Work Package Leader		15												
-	contributorType	Other		16												
-	authorIdentifierScheme	ORCID		0												
-	authorIdentifierScheme	ISNI		1												
-	authorIdentifierScheme	LCNA		2												
-	authorIdentifierScheme	VIAF		3												
-	authorIdentifierScheme	GND		4												
-	authorIdentifierScheme	DAI		5												
-	authorIdentifierScheme	ResearcherID		6												
-	authorIdentifierScheme	ScopusID		7												
-	language	Abkhaz		0												
-	language	Afar		1	aar	aa										
-	language	Afrikaans		2	afr	af										
-	language	Akan		3	aka	ak										
-	language	Albanian		4	sqi	alb	sq									
-	language	Amharic		5	amh	am										
-	language	Arabic		6	ara	ar										
-	language	Aragonese		7	arg	an										
-	language	Armenian		8	hye	arm	hy									
-	language	Assamese		9	asm	as										
-	language	Avaric		10	ava	av										
-	language	Avestan		11	ave	ae										
-	language	Aymara		12	aym	ay										
-	language	Azerbaijani		13	aze	az										
-	language	Bambara		14	bam	bm										
-	language	Bashkir		15	bak	ba										
-	language	Basque		16	eus	baq	eu									
-	language	Belarusian		17	bel	be										
-	language	Bengali, Bangla		18	ben	bn										
-	language	Bihari		19	bih	bh										
-	language	Bislama		20	bis	bi										
-	language	Bosnian		21	bos	bs										
-	language	Breton		22	bre	br										
-	language	Bulgarian		23	bul	bg										
-	language	Burmese		24	mya	bur	my									
-	language	Catalan,Valencian		25	cat ca											
-	language	Chamorro		26	cha	ch										
-	language	Chechen		27	che	ce										
-	language	Chichewa, Chewa, Nyanja		28	nya	ny										
-	language	Chinese		29	zho	chi	zh									
-	language	Chuvash		30	chv	cv										
-	language	Cornish		31	cor	kw										
-	language	Corsican		32	cos	co										
-	language	Cree		33	cre	cr										
-	language	Croatian		34	hrv	src	hr									
-	language	Czech		35	ces	cze	cs									
-	language	Danish		36	dan	da										
-	language	Divehi, Dhivehi, Maldivian		37	div	dv										
-	language	Dutch		38	nld	dut	nl									
-	language	Dzongkha		39	dzo	dz										
-	language	English		40	eng	en										
-	language	Esperanto		41	epo	eo										
-	language	Estonian		42	est	et										
-	language	Ewe		43	ewe	ee										
-	language	Faroese		44	fao	fo										
-	language	Fijian		45	fij	fj										
-	language	Finnish		46	fin	fi										
-	language	French		47	fra	fre	fr									
-	language	Fula, Fulah, Pulaar, Pular		48	ful	ff										
-	language	Galician		49	glg	gl										
-	language	Georgian		50	kat	geo	ka									
-	language	German		51	deu	ger	de									
-	language	Greek (modern)		52	gre	ell	el									
-	language	Guaraní		53	grn	gn										
-	language	Gujarati		54	guj	gu										
-	language	Haitian, Haitian Creole		55	hat ht											
-	language	Hausa		56	hau	ha										
-	language	Hebrew (modern)		57	heb	he										
-	language	Herero		58	her	hz										
-	language	Hindi		59	hin	hi										
-	language	Hiri Motu		60	hmo	ho										
-	language	Hungarian		61	hun	hu										
-	language	Interlingua		62	ina	ia										
-	language	Indonesian		63	ind	id										
-	language	Interlingue		64	ile	ie										
-	language	Irish		65	gle	ga										
-	language	Igbo		66	ibo	ig										
-	language	Inupiaq		67	ipk	ik										
-	language	Ido		68	ido	io										
-	language	Icelandic		69	isl	ice	is									
-	language	Italian		70	ita	it										
-	language	Inuktitut		71	iku	iu										
-	language	Japanese		72	jpn	ja										
-	language	Javanese		73	jav	jv										
-	language	Kalaallisut, Greenlandic		74	kal	kl										
-	language	Kannada		75	kan	kn										
-	language	Kanuri		76	kau	kr										
-	language	Kashmiri		77	kas	ks										
-	language	Kazakh		78	kaz	kk										
-	language	Khmer		79	khm	km										
-	language	Kikuyu, Gikuyu		80	kik	ki										
-	language	Kinyarwanda		81	kin	rw										
-	language	Kyrgyz		82												
-	language	Komi		83	kom	kv										
-	language	Kongo		84	kon	kg										
-	language	Korean		85	kor	ko										
-	language	Kurdish		86	kur	ku										
-	language	Kwanyama, Kuanyama		87	kua	kj										
-	language	Latin		88	lat	la										
-	language	Luxembourgish, Letzeburgesch		89	ltz	lb										
-	language	Ganda		90	lug	lg										
-	language	Limburgish, Limburgan, Limburger		91	lim	li										
-	language	Lingala		92	lin	ln										
-	language	Lao		93	lao	lo										
-	language	Lithuanian		94	lit	lt										
-	language	Luba-Katanga		95	lub	lu										
-	language	Latvian		96	lav	lv										
-	language	Manx		97	glv	gv										
-	language	Macedonian		98	mkd	mac	mk									
-	language	Malagasy		99	mlg	mg										
-	language	Malay		100	may	msa	ms									
-	language	Malayalam		101	mal	ml										
-	language	Maltese		102	mlt	mt										
-	language	M�ori		103	mao	mri	mi									
-	language	Marathi (Mar�ṭhī)		104	mar	mr										
-	language	Marshallese		105	mah	mh										
-	language	Mixtepec Mixtec		106	mix											
-	language	Mongolian		107	mon	mn										
-	language	Nauru		108	nau	na										
-	language	Navajo, Navaho		109	nav	nv										
-	language	Northern Ndebele		110	nde	nd										
-	language	Nepali		111	nep	ne										
-	language	Ndonga		112	ndo	ng										
-	language	Norwegian Bokmål		113	nob	nb										
-	language	Norwegian Nynorsk		114	nno	nn										
-	language	Norwegian		115	nor	no										
-	language	Nuosu		116												
-	language	Southern Ndebele		117	nbl	nr										
-	language	Occitan		118	oci	oc										
-	language	Ojibwe, Ojibwa		119	oji	oj										
-	language	Old Church Slavonic,Church Slavonic,Old Bulgarian		120	chu	cu										
-	language	Oromo		121	orm	om										
-	language	Oriya		122	ori	or										
-	language	Ossetian, Ossetic		123	oss	os										
-	language	Panjabi, Punjabi		124	pan	pa										
-	language	P�li		125	pli	pi										
-	language	Persian (Farsi)		126	per	fas	fa									
-	language	Polish		127	pol	pl										
-	language	Pashto, Pushto		128	pus	ps										
-	language	Portuguese		129	por	pt										
-	language	Quechua		130	que	qu										
-	language	Romansh		131	roh	rm										
-	language	Kirundi		132	run	rn										
-	language	Romanian		133	ron	rum	ro									
-	language	Russian		134	rus	ru										
-	language	Sanskrit (Sa�skṛta)		135	san	sa										
-	language	Sardinian		136	srd	sc										
-	language	Sindhi		137	snd	sd										
-	language	Northern Sami		138	sme	se										
-	language	Samoan		139	smo	sm										
-	language	Sango		140	sag	sg										
-	language	Serbian		141	srp	scc	sr									
-	language	Scottish Gaelic, Gaelic		142	gla	gd										
-	language	Shona		143	sna	sn										
-	language	Sinhala, Sinhalese		144	sin	si										
-	language	Slovak		145	slk	slo	sk									
-	language	Slovene		146	slv	sl										
-	language	Somali		147	som	so										
-	language	Southern Sotho		148	sot	st										
-	language	Spanish, Castilian		149	spa	es										
-	language	Sundanese		150	sun	su										
-	language	Swahili		151	swa	sw										
-	language	Swati		152	ssw	ss										
-	language	Swedish		153	swe	sv										
-	language	Tamil		154	tam	ta										
-	language	Telugu		155	tel	te										
-	language	Tajik		156	tgk	tg										
-	language	Thai		157	tha	th										
-	language	Tigrinya		158	tir	ti										
-	language	Tibetan Standard, Tibetan, Central		159	tib	bod	bo									
-	language	Turkmen		160	tuk	tk										
-	language	Tagalog		161	tgl	tl										
-	language	Tswana		162	tsn	tn										
-	language	Tonga (Tonga Islands)		163	ton	to										
-	language	Turkish		164	tur	tr										
-	language	Tsonga		165	tso	ts										
-	language	Tatar		166	tat	tt										
-	language	Twi		167	twi	tw										
-	language	Tahitian		168	tah	ty										
-	language	Uyghur, Uighur		169	uig	ug										
-	language	Ukrainian		170	ukr	uk										
-	language	Urdu		171	urd	ur										
-	language	Uzbek		172	uzb	uz										
-	language	Venda		173	ven	ve										
-	language	Vietnamese		174	vie	vi										
-	language	Volapük		175	vol	vo										
-	language	Walloon		176	wln	wa										
-	language	Welsh		177	cym	wel	cy									
-	language	Wolof		178	wol	wo										
-	language	Western Frisian		179	fry	fy										
-	language	Xhosa		180	xho	xh										
-	language	Yiddish		181	yid	yi										
-	language	Yoruba		182	yor	yo										
-	language	Zhuang, Chuang		183	zha	za										
-	language	Zulu		184	zul	zu										
-	language	Not applicable		185												
+#controlledVocabulary	DatasetField	Value	identifier	displayOrder											
+	subject	Agricultural Sciences	D01	0											
+	subject	Arts and Humanities	D0	1											
+	subject	Astronomy and Astrophysics	D1	2											
+	subject	Business and Management	D2	3											
+	subject	Chemistry	D3	4
+	subject	Computer and Information Science	D7	5									
+	subject	Earth and Environmental Sciences	D4	6											
+	subject	Engineering	D5	7
+	subject	Law	D8	8	
+	subject	Mathematical Sciences	D9	9																	
+	subject	Medicine, Health and Life Sciences	D6	10											
+	subject	Physics	D10	11											
+	subject	Social Sciences	D11	12											
+	subject	Other	D12	13											
+	publicationIDType	ark		0											
+	publicationIDType	arXiv		1											
+	publicationIDType	bibcode		2								
+	publicationIDType	cstr		3											
+	publicationIDType	doi		4											
+	publicationIDType	ean13		5											
+	publicationIDType	eissn		6											
+	publicationIDType	handle		7											
+	publicationIDType	isbn		8											
+	publicationIDType	issn		9											
+	publicationIDType	istc		10											
+	publicationIDType	lissn		11											
+	publicationIDType	lsid		12											
+	publicationIDType	pmid		13											
+	publicationIDType	purl		14											
+	publicationIDType	upc		15											
+	publicationIDType	url		16	
+	publicationIDType	urn		17											
+	publicationIDType	DASH-NRS		18		
+	contributorType	Data Collector		0											
+	contributorType	Data Curator		1											
+	contributorType	Data Manager		2											
+	contributorType	Editor		3											
+	contributorType	Funder		4											
+	contributorType	Hosting Institution		5											
+	contributorType	Project Leader		6											
+	contributorType	Project Manager		7											
+	contributorType	Project Member		8											
+	contributorType	Related Person		9											
+	contributorType	Researcher		10											
+	contributorType	Research Group		11											
+	contributorType	Rights Holder		12											
+	contributorType	Sponsor		13											
+	contributorType	Supervisor		14											
+	contributorType	Work Package Leader		15											
+	contributorType	Other		16											
+	authorIdentifierScheme	ORCID		0											
+	authorIdentifierScheme	ISNI		1											
+	authorIdentifierScheme	LCNA		2											
+	authorIdentifierScheme	VIAF		3											
+	authorIdentifierScheme	GND		4											
+	authorIdentifierScheme	DAI		5
+	authorIdentifierScheme	ResearcherID		6
+	authorIdentifierScheme	ScopusID		7
+	language	Abkhaz		0
+	language	Afar		1	aar	aa
+	language	Afrikaans		2	afr	af
+	language	Akan		3	aka	ak
+	language	Albanian		4	sqi	alb	sq
+	language	Amharic		5	amh	am
+	language	Arabic		6	ara	ar
+	language	Aragonese		7	arg	an
+	language	Armenian		8	hye	arm	hy
+	language	Assamese		9	asm	as
+	language	Avaric		10	ava	av
+	language	Avestan		11	ave	ae
+	language	Aymara		12	aym	ay
+	language	Azerbaijani		13	aze	az
+	language	Bambara		14	bam	bm
+	language	Bashkir		15	bak	ba
+	language	Basque		16	eus	baq	eu
+	language	Belarusian		17	bel	be
+	language	Bengali, Bangla		18	ben	bn
+	language	Bihari		19	bih	bh
+	language	Bislama		20	bis	bi
+	language	Bosnian		21	bos	bs
+	language	Breton		22	bre	br
+	language	Bulgarian		23	bul	bg
+	language	Burmese		24	mya	bur	my
+	language	Catalan,Valencian		25	cat ca
+	language	Chamorro		26	cha	ch
+	language	Chechen		27	che	ce
+	language	Chichewa, Chewa, Nyanja		28	nya	ny
+	language	Chinese		29	zho	chi	zh
+	language	Chuvash		30	chv	cv
+	language	Cornish		31	cor	kw
+	language	Corsican		32	cos	co
+	language	Cree		33	cre	cr
+	language	Croatian		34	hrv	src	hr
+	language	Czech		35	ces	cze	cs
+	language	Danish		36	dan	da
+	language	Divehi, Dhivehi, Maldivian		37	div	dv
+	language	Dutch		38	nld	dut	nl
+	language	Dzongkha		39	dzo	dz
+	language	English		40	eng	en
+	language	Esperanto		41	epo	eo
+	language	Estonian		42	est	et
+	language	Ewe		43	ewe	ee
+	language	Faroese		44	fao	fo
+	language	Fijian		45	fij	fj
+	language	Finnish		46	fin	fi
+	language	French		47	fra	fre	fr
+	language	Fula, Fulah, Pulaar, Pular		48	ful	ff
+	language	Galician		49	glg	gl
+	language	Georgian		50	kat	geo	ka
+	language	German		51	deu	ger	de
+	language	Greek (modern)		52	gre	ell	el
+	language	Guaraní		53	grn	gn
+	language	Gujarati		54	guj	gu
+	language	Haitian, Haitian Creole		55	hat ht
+	language	Hausa		56	hau	ha
+	language	Hebrew (modern)		57	heb	he
+	language	Herero		58	her	hz
+	language	Hindi		59	hin	hi
+	language	Hiri Motu		60	hmo	ho
+	language	Hungarian		61	hun	hu
+	language	Interlingua		62	ina	ia
+	language	Indonesian		63	ind	id
+	language	Interlingue		64	ile	ie
+	language	Irish		65	gle	ga
+	language	Igbo		66	ibo	ig
+	language	Inupiaq		67	ipk	ik
+	language	Ido		68	ido	io
+	language	Icelandic		69	isl	ice	is
+	language	Italian		70	ita	it
+	language	Inuktitut		71	iku	iu
+	language	Japanese		72	jpn	ja
+	language	Javanese		73	jav	jv
+	language	Kalaallisut, Greenlandic		74	kal	kl
+	language	Kannada		75	kan	kn
+	language	Kanuri		76	kau	kr
+	language	Kashmiri		77	kas	ks
+	language	Kazakh		78	kaz	kk
+	language	Khmer		79	khm	km
+	language	Kikuyu, Gikuyu		80	kik	ki
+	language	Kinyarwanda		81	kin	rw
+	language	Kyrgyz		82											
+	language	Komi		83	kom	kv
+	language	Kongo		84	kon	kg
+	language	Korean		85	kor	ko
+	language	Kurdish		86	kur	ku
+	language	Kwanyama, Kuanyama		87	kua	kj
+	language	Latin		88	lat	la
+	language	Luxembourgish, Letzeburgesch		89	ltz	lb
+	language	Ganda		90	lug	lg
+	language	Limburgish, Limburgan, Limburger		91	lim	li
+	language	Lingala		92	lin	ln
+	language	Lao		93	lao	lo
+	language	Lithuanian		94	lit	lt
+	language	Luba-Katanga		95	lub	lu
+	language	Latvian		96	lav	lv
+	language	Manx		97	glv	gv
+	language	Macedonian		98	mkd	mac	mk
+	language	Malagasy		99	mlg	mg
+	language	Malay		100	may	msa	ms
+	language	Malayalam		101	mal	ml
+	language	Maltese		102	mlt	mt
+	language	M�ori		103	mao	mri	mi
+	language	Marathi (Mar�ṭhī)		104	mar	mr
+	language	Marshallese		105	mah	mh
+	language	Mixtepec Mixtec		106	mix
+	language	Mongolian		107	mon	mn
+	language	Nauru		108	nau	na
+	language	Navajo, Navaho		109	nav	nv
+	language	Northern Ndebele		110	nde	nd
+	language	Nepali		111	nep	ne
+	language	Ndonga		112	ndo	ng
+	language	Norwegian Bokmål		113	nob	nb
+	language	Norwegian Nynorsk		114	nno	nn
+	language	Norwegian		115	nor	no
+	language	Nuosu		116											
+	language	Southern Ndebele		117	nbl	nr
+	language	Occitan		118	oci	oc
+	language	Ojibwe, Ojibwa		119	oji	oj
+	language	Old Church Slavonic,Church Slavonic,Old Bulgarian		120	chu	cu
+	language	Oromo		121	orm	om
+	language	Oriya		122	ori	or
+	language	Ossetian, Ossetic		123	oss	os
+	language	Panjabi, Punjabi		124	pan	pa
+	language	P�li		125	pli	pi
+	language	Persian (Farsi)		126	per	fas	fa
+	language	Polish		127	pol	pl
+	language	Pashto, Pushto		128	pus	ps
+	language	Portuguese		129	por	pt
+	language	Quechua		130	que	qu
+	language	Romansh		131	roh	rm
+	language	Kirundi		132	run	rn
+	language	Romanian		133	ron	rum	ro
+	language	Russian		134	rus	ru
+	language	Sanskrit (Sa�skṛta)		135	san	sa
+	language	Sardinian		136	srd	sc
+	language	Sindhi		137	snd	sd
+	language	Northern Sami		138	sme	se
+	language	Samoan		139	smo	sm
+	language	Sango		140	sag	sg
+	language	Serbian		141	srp	scc	sr
+	language	Scottish Gaelic, Gaelic		142	gla	gd
+	language	Shona		143	sna	sn
+	language	Sinhala, Sinhalese		144	sin	si
+	language	Slovak		145	slk	slo	sk
+	language	Slovene		146	slv	sl
+	language	Somali		147	som	so
+	language	Southern Sotho		148	sot	st
+	language	Spanish, Castilian		149	spa	es
+	language	Sundanese		150	sun	su
+	language	Swahili		151	swa	sw
+	language	Swati		152	ssw	ss
+	language	Swedish		153	swe	sv
+	language	Tamil		154	tam	ta
+	language	Telugu		155	tel	te
+	language	Tajik		156	tgk	tg
+	language	Thai		157	tha	th
+	language	Tigrinya		158	tir	ti
+	language	Tibetan Standard, Tibetan, Central		159	tib	bod	bo
+	language	Turkmen		160	tuk	tk
+	language	Tagalog		161	tgl	tl
+	language	Tswana		162	tsn	tn
+	language	Tonga (Tonga Islands)		163	ton	to
+	language	Turkish		164	tur	tr
+	language	Tsonga		165	tso	ts
+	language	Tatar		166	tat	tt
+	language	Twi		167	twi	tw
+	language	Tahitian		168	tah	ty
+	language	Uyghur, Uighur		169	uig	ug
+	language	Ukrainian		170	ukr	uk
+	language	Urdu		171	urd	ur
+	language	Uzbek		172	uzb	uz
+	language	Venda		173	ven	ve
+	language	Vietnamese		174	vie	vi
+	language	Volapük		175	vol	vo
+	language	Walloon		176	wln	wa
+	language	Welsh		177	cym	wel	cy
+	language	Wolof		178	wol	wo
+	language	Western Frisian		179	fry	fy
+	language	Xhosa		180	xho	xh
+	language	Yiddish		181	yid	yi
+	language	Yoruba		182	yor	yo
+	language	Zhuang, Chuang		183	zha	za
+	language	Zulu		184	zul	zu
+	language	Not applicable		185											
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index f7f0e30ea6e..458803e0c92 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -1337,6 +1337,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
         List<HashSet<FieldDTO>> producers = new ArrayList<>();
         List<HashSet<FieldDTO>> grants = new ArrayList<>();
         List<HashSet<FieldDTO>> software = new ArrayList<>();
+        List<String> prodPlac = new ArrayList<>();
 
         for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
             if (event == XMLStreamConstants.START_ELEMENT) {
@@ -1352,9 +1353,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
                 } else if (xmlr.getLocalName().equals("prodDate")) {
                     citation.getFields().add(FieldDTO.createPrimitiveFieldDTO("productionDate", parseDate(xmlr, "prodDate")));
                 } else if (xmlr.getLocalName().equals("prodPlac")) {
-                    List<String> prodPlac = new ArrayList<>();
                     prodPlac.add(parseText(xmlr, "prodPlac"));
-                    citation.getFields().add(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.productionPlace, prodPlac));
                 } else if (xmlr.getLocalName().equals("software")) {
                     HashSet<FieldDTO> set = new HashSet<>();
                     addToSet(set,"softwareVersion", xmlr.getAttributeValue(null, "version"));
@@ -1387,6 +1386,9 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
                     if (producers.size()>0) {
                         citation.getFields().add(FieldDTO.createMultipleCompoundFieldDTO("producer", producers));
                     }
+                    if (prodPlac.size() > 0) {
+                        citation.getFields().add(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.productionPlace, prodPlac));
+                    }
                     return;
                 }
             }
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index eb53473d4d9..4ea90ea6199 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -895,8 +895,13 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT
                 }
             }
         }
-        writeFullElement(xmlw, "prodDate", dto2Primitive(version, DatasetFieldConstant.productionDate));    
-        writeFullElement(xmlw, "prodPlac", dto2Primitive(version, DatasetFieldConstant.productionPlace));
+        writeFullElement(xmlw, "prodDate", dto2Primitive(version, DatasetFieldConstant.productionDate));
+
+        FieldDTO  prodPlac = dto2FieldDTO( version, DatasetFieldConstant.productionPlace, "citation"  );
+        if (prodPlac != null) {
+            writeMultipleElement(xmlw, "prodPlac", prodPlac, null);
+        }
+
         writeSoftwareElement(xmlw, version);
   
         writeGrantElement(xmlw, version);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
index 2ba06314ddb..14185b97e9e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
@@ -758,7 +758,7 @@ public void testLoadMetadataBlock_NoErrorPath() {
         assertEquals(1, data.size());
         List<Map<String, String>> addedElements = data.get("added");
         //Note -test depends on the number of elements in the production citation block, so any changes to the # of elements there can break this test
-        assertEquals(323, addedElements.size());
+        assertEquals(322, addedElements.size());
 
         Map<String, Integer> statistics = new HashMap<>();
         for (Map<String, String> unit : addedElements) {
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
index 96f058b1b02..822623f721a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
+++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
@@ -409,9 +409,9 @@
           },
           {
             "typeName": "productionPlace",
-            "multiple": false,
+            "multiple": true,
             "typeClass": "primitive",
-            "value": "ProductionPlace"
+            "value": ["ProductionPlace"]
           },
           {
             "typeName": "contributor",

From 89dbc7a87407a095670fa66dda87616b8500959d Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Thu, 16 Mar 2023 13:19:33 -0400
Subject: [PATCH 039/922] restore back

---
 src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
index 14185b97e9e..2ba06314ddb 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
@@ -758,7 +758,7 @@ public void testLoadMetadataBlock_NoErrorPath() {
         assertEquals(1, data.size());
         List<Map<String, String>> addedElements = data.get("added");
         //Note -test depends on the number of elements in the production citation block, so any changes to the # of elements there can break this test
-        assertEquals(322, addedElements.size());
+        assertEquals(323, addedElements.size());
 
         Map<String, Integer> statistics = new HashMap<>();
         for (Map<String, String> unit : addedElements) {

From 10a475317771227b23263170ad3c97232764d14d Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 20 Mar 2023 10:48:20 -0400
Subject: [PATCH 040/922] An extra check, to disable the upload component right
 away, if the quota is already full/exceeded. #9361

---
 src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 4 ++++
 src/main/webapp/editFilesFragment.xhtml                       | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index 420642f2fa5..c39e6f62ce2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -542,6 +542,10 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo
         saveEnabled = true;
         return null;
     }
+    
+    public boolean isQuotaExceeded() {
+        return systemConfig.isStorageQuotasEnforced() && datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset).getRemainingQuotaInBytes() == 0;
+    }
 
     public String init() {
         // default mode should be EDIT
diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml
index 77f7aab2f76..834ca597892 100644
--- a/src/main/webapp/editFilesFragment.xhtml
+++ b/src/main/webapp/editFilesFragment.xhtml
@@ -148,7 +148,7 @@
                                   dragDropSupport="true"
                                   auto="#{!(systemConfig.directUploadEnabled(EditDatafilesPage.dataset))}"
                                   multiple="#{datasetPage || EditDatafilesPage.allowMultipleFileUpload()}"
-                                  disabled="#{lockedFromEdits ||  !(datasetPage || EditDatafilesPage.showFileUploadComponent()) }"
+                                  disabled="#{lockedFromEdits ||  !(datasetPage || EditDatafilesPage.showFileUploadComponent()) || EditDatafilesPage.isQuotaExceeded()}"
                                   listener="#{EditDatafilesPage.handleFileUpload}" 
                                   process="filesTable" 
                                   update=":datasetForm:filesTable, @([id$=filesButtons])"

From 098de49c8ac14478ad01d9aaa2f820e3d9ab744d Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Fri, 21 Apr 2023 16:10:41 +0200
Subject: [PATCH 041/922] reverted SystemConfig.java changes

---
 .../iq/dataverse/util/SystemConfig.java       | 299 ++++++++++--------
 1 file changed, 160 insertions(+), 139 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 1764abf4478..c989add6e3d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -73,7 +73,7 @@ public class SystemConfig {
      * The default number of datafiles that we allow to be created through 
      * zip file upload.
      */
-    private static final int defaultZipUploadFilesLimit = 1000;
+    private static final int defaultZipUploadFilesLimit = 1000; 
     public static final long defaultZipDownloadLimit = 104857600L; // 100MB
     private static final int defaultMultipleUploadFilesLimit = 1000;
     private static final int defaultLoginSessionTimeout = 480; // = 8 hours
@@ -81,21 +81,21 @@ public class SystemConfig {
     private String buildNumber = null;
     
     private static final String JVM_TIMER_SERVER_OPTION = "dataverse.timerServer";
-
-    private static final long DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT = 5000L;
+    
+    private static final long DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT = 5000L; 
     private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE = 3000000L; // 3 MB
     private static final long DEFAULT_THUMBNAIL_SIZE_LIMIT_PDF = 1000000L; // 1 MB
-
+    
     public final static String DEFAULTCURATIONLABELSET = "DEFAULT";
     public final static String CURATIONLABELSDISABLED = "DISABLED";
-
+    
     public String getVersion() {
         return getVersion(false);
     }
-
+    
     // The return value is a "prviate static String", that should be initialized
-    // once, on the first call (see the code below)... But this is a @Stateless
-    // bean... so that would mean "once per thread"? - this would be a prime
+    // once, on the first call (see the code below)... But this is a @Stateless 
+    // bean... so that would mean "once per thread"? - this would be a prime 
     // candidate for being moved into some kind of an application-scoped caching
     // service... some CachingService @Singleton - ? (L.A. 5.8)
     public String getVersion(boolean withBuildNumber) {
@@ -157,15 +157,15 @@ public String getSolrHostColonPort() {
 
     public boolean isProvCollectionEnabled() {
         String provCollectionEnabled = settingsService.getValueForKey(SettingsServiceBean.Key.ProvCollectionEnabled, null);
-        if ("true".equalsIgnoreCase(provCollectionEnabled)) {
+        if("true".equalsIgnoreCase(provCollectionEnabled)){         
             return true;
         }
         return false;
 
     }
-
+    
     public int getMetricsCacheTimeoutMinutes() {
-        int defaultValue = 10080; // one week in minutes
+        int defaultValue = 10080; //one week in minutes
         SettingsServiceBean.Key key = SettingsServiceBean.Key.MetricsCacheTimeoutMinutes;
         String metricsCacheTimeString = settingsService.getValueForKey(key);
         if (metricsCacheTimeString != null) {
@@ -183,7 +183,7 @@ public int getMetricsCacheTimeoutMinutes() {
         }
         return defaultValue;
     }
-
+    
     public int getMinutesUntilConfirmEmailTokenExpires() {
         final int minutesInOneDay = 1440;
         final int reasonableDefault = minutesInOneDay;
@@ -201,9 +201,34 @@ public int getMinutesUntilConfirmEmailTokenExpires() {
                 logger.info("Returning " + reasonableDefault + " for " + key + " because value must be an integer greater than zero, not \"" + valueFromDatabase + "\".");
             }
         }
+        logger.fine("Returning " + reasonableDefault + " for " + key);
         return reasonableDefault;
     }
 
+    /**
+     * The number of minutes for which a password reset token is valid. Can be
+     * overridden by {@link #PASSWORD_RESET_TIMEOUT_IN_MINUTES}.
+     */
+    public static int getMinutesUntilPasswordResetTokenExpires() {
+        final int reasonableDefault = 60;
+        String configuredValueAsString = System.getProperty(PASSWORD_RESET_TIMEOUT_IN_MINUTES);
+        if (configuredValueAsString != null) {
+            int configuredValueAsInteger = 0;
+            try {
+                configuredValueAsInteger = Integer.parseInt(configuredValueAsString);
+                if (configuredValueAsInteger > 0) {
+                    return configuredValueAsInteger;
+                } else {
+                    logger.info(PASSWORD_RESET_TIMEOUT_IN_MINUTES + " is configured as a negative number \"" + configuredValueAsInteger + "\". Using default value instead: " + reasonableDefault);
+                    return reasonableDefault;
+                }
+            } catch (NumberFormatException ex) {
+                logger.info("Unable to convert " + PASSWORD_RESET_TIMEOUT_IN_MINUTES + " from \"" + configuredValueAsString + "\" into an integer value: " + ex + ". Using default value " + reasonableDefault);
+            }
+        }
+        return reasonableDefault;
+    }
+    
     /**
      * Lookup (or construct) the designated URL of this instance from configuration.
      *
@@ -226,6 +251,7 @@ public String getDataverseSiteUrl() {
      *
      * Can be defined as a complete URL via <code>dataverse.siteUrl</code>; or derived from the hostname
      * <code>dataverse.fqdn</code> and HTTPS. If none of these options is set, defaults to the
+     * {@link InetAddress#getLocalHost} and HTTPS.
      *
      * NOTE: This method does not provide any validation.
      * TODO: The behaviour of this method is subject to a later change, see
@@ -257,13 +283,12 @@ public static String getDataverseSiteUrlStatic() {
             return null;
         }
     }
-
+    
     /**
-     * URL Tracking:
+     * URL Tracking: 
      */
     public String getPageURLWithQueryString() {
-        return PrettyContext.getCurrentInstance().getRequestURL().toURL()
-                + PrettyContext.getCurrentInstance().getRequestQueryString().toQueryString();
+        return PrettyContext.getCurrentInstance().getRequestURL().toURL() + PrettyContext.getCurrentInstance().getRequestQueryString().toQueryString();
     }
 
     public String getGuidesBaseUrl() {
@@ -322,44 +347,44 @@ public static int getIntLimitFromStringOrDefault(String limitSetting, Integer de
 
     /**
      * Download-as-zip size limit.
-     * returns defaultZipDownloadLimit if not specified;
-     * set to -1 to disable zip downloads.
+     * returns defaultZipDownloadLimit if not specified; 
+     * set to -1 to disable zip downloads. 
      */
     public long getZipDownloadLimit() {
         String zipLimitOption = settingsService.getValueForKey(SettingsServiceBean.Key.ZipDownloadLimit);
         return getLongLimitFromStringOrDefault(zipLimitOption, defaultZipDownloadLimit);
     }
-
+    
     public int getZipUploadFilesLimit() {
         String limitOption = settingsService.getValueForKey(SettingsServiceBean.Key.ZipUploadFilesLimit);
         return getIntLimitFromStringOrDefault(limitOption, defaultZipUploadFilesLimit);
     }
-
+    
     /**
-     * Session timeout, in minutes.
+     * Session timeout, in minutes. 
      * (default value provided)
      */
     public int getLoginSessionTimeout() {
         return getIntLimitFromStringOrDefault(
-                settingsService.getValueForKey(SettingsServiceBean.Key.LoginSessionTimeout),
-                defaultLoginSessionTimeout);
+                settingsService.getValueForKey(SettingsServiceBean.Key.LoginSessionTimeout), 
+                defaultLoginSessionTimeout); 
     }
-
+    
     /*
     `   the number of files the GUI user is allowed to upload in one batch, 
         via drag-and-drop, or through the file select dialog
-     */
+    */
     public int getMultipleUploadFilesLimit() {
         String limitOption = settingsService.getValueForKey(SettingsServiceBean.Key.MultipleUploadFilesLimit);
         return getIntLimitFromStringOrDefault(limitOption, defaultMultipleUploadFilesLimit);
     }
-
+    
     public long getGuestbookResponsesPageDisplayLimit() {
         String limitSetting = settingsService.getValueForKey(SettingsServiceBean.Key.GuestbookResponsesPageDisplayLimit);
         return getLongLimitFromStringOrDefault(limitSetting, DEFAULT_GUESTBOOK_RESPONSES_DISPLAY_LIMIT);
     }
-
-    public long getUploadLogoSizeLimit() {
+    
+    public long getUploadLogoSizeLimit(){
         return 500000;
     }
 
@@ -372,10 +397,10 @@ public long getThumbnailSizeLimitPDF() {
     }
 
     public static long getThumbnailSizeLimit(String type) {
-        String option = null;
-
-        // get options via jvm options
-
+        String option = null; 
+        
+        //get options via jvm options
+        
         if ("Image".equals(type)) {
             option = System.getProperty("dataverse.dataAccess.thumbnail.image.limit");
             return getLongLimitFromStringOrDefault(option, DEFAULT_THUMBNAIL_SIZE_LIMIT_IMAGE);
@@ -387,19 +412,19 @@ public static long getThumbnailSizeLimit(String type) {
         // Zero (0) means no limit.
         return getLongLimitFromStringOrDefault(option, 0L);
     }
-
+    
     public boolean isThumbnailGenerationDisabledForType(String type) {
         return getThumbnailSizeLimit(type) == -1l;
     }
-
+    
     public boolean isThumbnailGenerationDisabledForImages() {
         return isThumbnailGenerationDisabledForType("Image");
     }
-
+    
     public boolean isThumbnailGenerationDisabledForPDF() {
         return isThumbnailGenerationDisabledForType("PDF");
     }
-
+    
     public String getApplicationTermsOfUse() {
         String language = BundleUtil.getCurrentLocale().getLanguage();
         String saneDefaultForAppTermsOfUse = BundleUtil.getStringFromBundle("system.app.terms");
@@ -407,9 +432,9 @@ public String getApplicationTermsOfUse() {
         // value, or as a better default than the saneDefaultForAppTermsOfUse if there
         // is no language-specific value
         String appTermsOfUse = settingsService.getValueForKey(SettingsServiceBean.Key.ApplicationTermsOfUse, saneDefaultForAppTermsOfUse);
-        // Now get the language-specific value if it exists
+        //Now get the language-specific value if it exists
         if (language != null && !language.equalsIgnoreCase(BundleUtil.getDefaultLocale().getLanguage())) {
-            appTermsOfUse = settingsService.getValueForKey(SettingsServiceBean.Key.ApplicationTermsOfUse, language, appTermsOfUse);
+            appTermsOfUse = settingsService.getValueForKey(SettingsServiceBean.Key.ApplicationTermsOfUse, language,	appTermsOfUse);
         }
         return appTermsOfUse;
     }
@@ -420,7 +445,7 @@ public String getApiTermsOfUse() {
         return apiTermsOfUse;
     }
 
-    // TODO:
+    // TODO: 
     // remove this method!
     // pages should be using settingsWrapper.get(":ApplicationPrivacyPolicyUrl") instead. -- 4.2.1
     public String getApplicationPrivacyPolicyUrl() {
@@ -439,10 +464,10 @@ public boolean isFilesOnDatasetPageFromSolr() {
         return settingsService.isTrueForKey(SettingsServiceBean.Key.FilesOnDatasetPageFromSolr, safeDefaultIfKeyNotFound);
     }
 
-    public Long getMaxFileUploadSizeForStore(String driverId) {
-        return settingsService.getValueForCompoundKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes, driverId);
-    }
-
+    public Long getMaxFileUploadSizeForStore(String driverId){
+         return settingsService.getValueForCompoundKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes, driverId);
+     }
+    
     public Integer getSearchHighlightFragmentSize() {
         String fragSize = settingsService.getValueForKey(SettingsServiceBean.Key.SearchHighlightFragmentSize);
         if (fragSize != null) {
@@ -456,12 +481,12 @@ public Integer getSearchHighlightFragmentSize() {
     }
 
     public long getTabularIngestSizeLimit() {
-        // This method will return the blanket ingestable size limit, if
-        // set on the system. I.e., the universal limit that applies to all
-        // tabular ingests, regardless of fromat:
-
-        String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit);
-
+        // This method will return the blanket ingestable size limit, if 
+        // set on the system. I.e., the universal limit that applies to all 
+        // tabular ingests, regardless of fromat: 
+        
+        String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.TabularIngestSizeLimit); 
+        
         if (limitEntry != null) {
             try {
                 Long sizeOption = new Long(limitEntry);
@@ -470,48 +495,48 @@ public long getTabularIngestSizeLimit() {
                 logger.warning("Invalid value for TabularIngestSizeLimit option? - " + limitEntry);
             }
         }
-        // -1 means no limit is set;
-        // 0 on the other hand would mean that ingest is fully disabled for
-        // tabular data.
-        return -1;
+        // -1 means no limit is set; 
+        // 0 on the other hand would mean that ingest is fully disabled for 
+        // tabular data. 
+        return -1; 
     }
-
+    
     public long getTabularIngestSizeLimit(String formatName) {
         // This method returns the size limit set specifically for this format name,
-        // if available, otherwise - the blanket limit that applies to all tabular
-        // ingests regardless of a format.
-
+        // if available, otherwise - the blanket limit that applies to all tabular 
+        // ingests regardless of a format. 
+        
         if (formatName == null || formatName.equals("")) {
-            return getTabularIngestSizeLimit();
+            return getTabularIngestSizeLimit(); 
         }
-
+        
         String limitEntry = settingsService.get(SettingsServiceBean.Key.TabularIngestSizeLimit.toString() + ":" + formatName); 
-
+                
         if (limitEntry != null) {
             try {
                 Long sizeOption = new Long(limitEntry);
                 return sizeOption;
             } catch (NumberFormatException nfe) {
-                logger.warning("Invalid value for TabularIngestSizeLimit:" + formatName + "? - " + limitEntry);
+                logger.warning("Invalid value for TabularIngestSizeLimit:" + formatName + "? - " + limitEntry );
             }
         }
-
-        return getTabularIngestSizeLimit();
+        
+        return getTabularIngestSizeLimit();        
     }
 
     public boolean isOAIServerEnabled() {
         boolean defaultResponse = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.OAIServerEnabled, defaultResponse);
     }
-
+    
     public void enableOAIServer() {
         settingsService.setValueForKey(SettingsServiceBean.Key.OAIServerEnabled, "true");
     }
-
+    
     public void disableOAIServer() {
         settingsService.deleteValueForKey(SettingsServiceBean.Key.OAIServerEnabled);
-    }
-
+    }   
+    
     public boolean isTimerServer() {
         String optionValue = System.getProperty(JVM_TIMER_SERVER_OPTION);
         if ("true".equalsIgnoreCase(optionValue)) {
@@ -579,12 +604,11 @@ public String getOAuth2CallbackUrl() {
         }
         return saneDefault;
     }
-
+    
     public boolean isShibPassiveLoginEnabled() {
         boolean defaultResponse = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.ShibPassiveLoginEnabled, defaultResponse);
     }
-
     public boolean isShibAttributeCharacterSetConversionEnabled() {
         boolean defaultResponse = true;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.ShibAttributeCharacterSetConversionEnabled, defaultResponse);
@@ -610,7 +634,7 @@ public String getPVDictionaries() {
     public int getPVGoodStrength() {
         // FIXME: Change this to 21 to match Harvard's requirements or implement a way to disable the rule (0 or -1) and have the default be disabled.
         int goodStrengthLength = 20;
-        // String _goodStrengthLength = System.getProperty("pv.goodstrength", settingsService.get(SettingsServiceBean.Key.PVGoodStrength.toString()));
+        //String _goodStrengthLength = System.getProperty("pv.goodstrength", settingsService.get(SettingsServiceBean.Key.PVGoodStrength.toString()));
         String _goodStrengthLength = settingsService.get(SettingsServiceBean.Key.PVGoodStrength.toString());
         try {
             goodStrengthLength = Integer.parseInt(_goodStrengthLength);
@@ -769,7 +793,8 @@ public static FileUploadMethods fromString(String text) {
         public String toString() {
             return text;
         }
-
+        
+        
     }
 
     /**
@@ -785,8 +810,8 @@ public enum FileDownloadMethods {
          */
         RSYNC("rsal/rsync"),
         NATIVE("native/http"),
-        GLOBUS("globus");
-
+        GLOBUS("globus")
+        ;
         private final String text;
 
         private FileDownloadMethods(final String text) {
@@ -808,28 +833,27 @@ public static FileUploadMethods fromString(String text) {
         public String toString() {
             return text;
         }
-
+        
     }
-
+    
     public enum DataFilePIDFormat {
         DEPENDENT("DEPENDENT"),
         INDEPENDENT("INDEPENDENT");
-
         private final String text;
 
         public String getText() {
             return text;
         }
-
-        private DataFilePIDFormat(final String text) {
+        
+        private DataFilePIDFormat(final String text){
             this.text = text;
         }
-
+        
         @Override
         public String toString() {
             return text;
         }
-
+        
     }
 
     /**
@@ -869,16 +893,16 @@ public String toString() {
 
     }
 
-    public boolean isPublicInstall() {
+    public boolean isPublicInstall(){
         boolean saneDefault = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.PublicInstall, saneDefault);
     }
-
-    public boolean isRsyncUpload() {
+    
+    public boolean isRsyncUpload(){
         return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), true);
     }
 
-    public boolean isGlobusUpload() {
+    public boolean isGlobusUpload(){
         return getMethodAvailable(FileUploadMethods.GLOBUS.toString(), true);
     }
     
@@ -887,30 +911,30 @@ public boolean isWebloaderUpload(){
     }
 
     // Controls if HTTP upload is enabled for both GUI and API.
-    public boolean isHTTPUpload() {
+    public boolean isHTTPUpload(){       
         return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), true);
     }
-
-    public boolean isRsyncOnly() {
+    
+    public boolean isRsyncOnly(){
         String downloadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.DownloadMethods);
-        if (downloadMethods == null) {
+        if(downloadMethods == null){
             return false;
         }
-        if (!downloadMethods.toLowerCase().equals(SystemConfig.FileDownloadMethods.RSYNC.toString())) {
+        if (!downloadMethods.toLowerCase().equals(SystemConfig.FileDownloadMethods.RSYNC.toString())){
             return false;
         }
         String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods);
-        if (uploadMethods == null) {
+        if (uploadMethods==null){
             return false;
         } else {
-            return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size() == 1 && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString());
+           return  Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size() == 1 && uploadMethods.toLowerCase().equals(SystemConfig.FileUploadMethods.RSYNC.toString());
         }
     }
-
+    
     public boolean isRsyncDownload() {
         return getMethodAvailable(SystemConfig.FileUploadMethods.RSYNC.toString(), false);
     }
-
+    
     public boolean isHTTPDownload() {
         return getMethodAvailable(SystemConfig.FileUploadMethods.NATIVE.toString(), false);
     }
@@ -918,14 +942,14 @@ public boolean isHTTPDownload() {
     public boolean isGlobusDownload() {
         return getMethodAvailable(FileUploadMethods.GLOBUS.toString(), false);
     }
-
+    
     public boolean isGlobusFileDownload() {
         return (isGlobusDownload() && settingsService.isTrueForKey(SettingsServiceBean.Key.GlobusSingleFileTransfer, false));
     }
 
     public List<String> getGlobusStoresList() {
-        String globusStores = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusStores, "");
-        return Arrays.asList(globusStores.split("\\s*,\\s*"));
+    String globusStores = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusStores, "");
+    return Arrays.asList(globusStores.split("\\s*,\\s*"));
     }
 
     private Boolean getMethodAvailable(String method, boolean upload) {
@@ -937,32 +961,31 @@ private Boolean getMethodAvailable(String method, boolean upload) {
             return Arrays.asList(methods.toLowerCase().split("\\s*,\\s*")).contains(method);
         }
     }
-
-    public Integer getUploadMethodCount() {
-        String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods);
-        if (uploadMethods == null) {
+    
+    public Integer getUploadMethodCount(){
+        String uploadMethods = settingsService.getValueForKey(SettingsServiceBean.Key.UploadMethods); 
+        if (uploadMethods==null){
             return 0;
         } else {
-            return Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size();
-        }
+           return  Arrays.asList(uploadMethods.toLowerCase().split("\\s*,\\s*")).size();
+        }       
     }
-
-    public boolean isDataFilePIDSequentialDependent() {
+    public boolean isDataFilePIDSequentialDependent(){
         String doiIdentifierType = settingsService.getValueForKey(SettingsServiceBean.Key.IdentifierGenerationStyle, "randomString");
         String doiDataFileFormat = settingsService.getValueForKey(SettingsServiceBean.Key.DataFilePIDFormat, "DEPENDENT");
-        if (doiIdentifierType.equals("storedProcGenerated") && doiDataFileFormat.equals("DEPENDENT")) {
+        if (doiIdentifierType.equals("storedProcGenerated") && doiDataFileFormat.equals("DEPENDENT")){
             return true;
         }
         return false;
     }
-
+    
     public int getPIDAsynchRegFileCount() {
         String fileCount = settingsService.getValueForKey(SettingsServiceBean.Key.PIDAsynchRegFileCount, "10");
         int retVal = 10;
         try {
             retVal = Integer.parseInt(fileCount);
-        } catch (NumberFormatException e) {
-            // if no number in the setting we'll return 10
+        } catch (NumberFormatException e) {           
+            //if no number in the setting we'll return 10
         }
         return retVal;
     }
@@ -976,13 +999,13 @@ public boolean isFilePIDsEnabled() {
         boolean safeDefaultIfKeyNotFound = true;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.FilePIDsEnabled, safeDefaultIfKeyNotFound);
     }
-
+    
     public boolean isIndependentHandleService() {
         boolean safeDefaultIfKeyNotFound = false;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.IndependentHandleService, safeDefaultIfKeyNotFound);
-
+    
     }
-
+    
     public String getHandleAuthHandle() {
         String handleAuthHandle = settingsService.getValueForKey(SettingsServiceBean.Key.HandleAuthHandle, null);
         return handleAuthHandle;
@@ -992,61 +1015,60 @@ public String getMDCLogPath() {
         String mDCLogPath = settingsService.getValueForKey(SettingsServiceBean.Key.MDCLogPath, null);
         return mDCLogPath;
     }
-
+    
     public boolean isDatafileValidationOnPublishEnabled() {
         boolean safeDefaultIfKeyNotFound = true;
         return settingsService.isTrueForKey(SettingsServiceBean.Key.FileValidationOnPublishEnabled, safeDefaultIfKeyNotFound);
     }
 
-    public boolean directUploadEnabled(DvObjectContainer container) {
-        // this method is used in UI only, therfore "dataverse.files." + driverId + ".upload-out-of-band" is not used here
-        return Boolean.parseBoolean(System.getProperty("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect"));
-    }
-
-    public String getDataCiteRestApiUrlString() {
-        // As of 5.0 the 'doi.dataciterestapiurlstring' is the documented jvm option. Prior versions used 'doi.mdcbaseurlstring' or were hardcoded to api.datacite.org, so the defaults are for backward compatibility.
+	public boolean directUploadEnabled(DvObjectContainer container) {
+    	return Boolean.getBoolean("dataverse.files." + container.getEffectiveStorageDriverId() + ".upload-redirect");
+	}
+	
+	public String getDataCiteRestApiUrlString() {
+		//As of 5.0 the 'doi.dataciterestapiurlstring' is the documented jvm option. Prior versions used 'doi.mdcbaseurlstring' or were hardcoded to api.datacite.org, so the defaults are for backward compatibility.
         return System.getProperty("doi.dataciterestapiurlstring", System.getProperty("doi.mdcbaseurlstring", "https://api.datacite.org"));
-    }
-
+	}
+        
     public boolean isExternalDataverseValidationEnabled() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataValidatorScript) != null;
-        // alternatively, we can also check if the script specified exists,
+        // alternatively, we can also check if the script specified exists, 
         // and is executable. -- ?
     }
-
+    
     public boolean isExternalDatasetValidationEnabled() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DatasetMetadataValidatorScript) != null;
-        // alternatively, we can also check if the script specified exists,
+        // alternatively, we can also check if the script specified exists, 
         // and is executable. -- ?
     }
-
+    
     public String getDataverseValidationExecutable() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataValidatorScript);
     }
-
+    
     public String getDatasetValidationExecutable() {
         return settingsService.getValueForKey(SettingsServiceBean.Key.DatasetMetadataValidatorScript);
     }
-
+    
     public String getDataverseValidationFailureMsg() {
         String defaultMessage = "This dataverse collection cannot be published because it has failed an external metadata validation test.";
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataPublishValidationFailureMsg, defaultMessage);
     }
-
+    
     public String getDataverseUpdateValidationFailureMsg() {
         String defaultMessage = "This dataverse collection cannot be updated because it has failed an external metadata validation test.";
         return settingsService.getValueForKey(SettingsServiceBean.Key.DataverseMetadataUpdateValidationFailureMsg, defaultMessage);
     }
-
+    
     public String getDatasetValidationFailureMsg() {
         String defaultMessage = "This dataset cannot be published because it has failed an external metadata validation test.";
         return settingsService.getValueForKey(SettingsServiceBean.Key.DatasetMetadataValidationFailureMsg, defaultMessage);
     }
-
+    
     public boolean isExternalValidationAdminOverrideEnabled() {
         return "true".equalsIgnoreCase(settingsService.getValueForKey(SettingsServiceBean.Key.ExternalValidationAdminOverride));
     }
-
+    
     public long getDatasetValidationSizeLimit() {
         String limitEntry = settingsService.getValueForKey(SettingsServiceBean.Key.DatasetChecksumValidationSizeLimit);
 
@@ -1076,7 +1098,6 @@ public long getFileValidationSizeLimit() {
         // -1 means no limit is set;
         return -1;
     }
-
     public Map<String, String[]> getCurationLabels() {
         Map<String, String[]> labelMap = new HashMap<String, String[]>();
         String setting = settingsService.getValueForKey(SettingsServiceBean.Key.AllowedCurationLabels, "");
@@ -1117,15 +1138,15 @@ public Map<String, String[]> getCurationLabels() {
         }
         return labelMap;
     }
-
+    
     public boolean isSignupDisabledForRemoteAuthProvider(String providerId) {
-        Boolean ret = settingsService.getValueForCompoundKeyAsBoolean(SettingsServiceBean.Key.AllowRemoteAuthSignUp, providerId);
-
-        // we default to false - i.e., "not disabled" if the setting is not present:
+        Boolean ret =  settingsService.getValueForCompoundKeyAsBoolean(SettingsServiceBean.Key.AllowRemoteAuthSignUp, providerId);
+        
+        // we default to false - i.e., "not disabled" if the setting is not present: 
         if (ret == null) {
-            return false;
+            return false; 
         }
-
-        return !ret;
+        
+        return !ret; 
     }
 }

From 364858af43fcce6a1cfb4decefa8215621d4f648 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 25 Apr 2023 17:27:51 -0400
Subject: [PATCH 042/922] Further reconciliation of the quotas branch with the
 zip optimization pr. (#9361)

---
 .../impl/CreateNewDataFilesCommand.java       | 111 +++++++++++++++---
 .../harvard/iq/dataverse/util/FileUtil.java   |  10 +-
 2 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 9f281f9446d..15304502232 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -41,10 +41,12 @@
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Enumeration;
 import java.util.List;
 import java.util.Optional;
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipFile;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import javax.enterprise.inject.spi.CDI;
@@ -224,6 +226,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 // DataFile objects from its contents:
             } else if (finalType.equals("application/zip")) {
 
+                ZipFile zipFile = null;
                 ZipInputStream unZippedIn = null;
                 ZipEntry zipEntry = null;
 
@@ -253,13 +256,88 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 	 }
                      */
 
+                    /** 
+                     * Perform a quick check for how many individual files are 
+                     * inside this zip archive. If it's above the limit, we can 
+                     * give up right away, without doing any unpacking. 
+                     * This should be a fairly inexpensive operation, we just need
+                     * to read the directory at the end of the file. 
+                     */
+                    
+                    if (charset != null) {
+                        zipFile = new ZipFile(tempFile.toFile(), charset);
+                    } else {
+                        zipFile = new ZipFile(tempFile.toFile());
+                    }
+                    /**
+                     * The ZipFile constructors above will throw ZipException - 
+                     * a type of IOException - if there's something wrong 
+                     * with this file as a zip. There's no need to intercept it
+                     * here, it will be caught further below, with other IOExceptions,
+                     * at which point we'll give up on trying to unpack it and
+                     * then attempt to save it as is.
+                     */
+
+                    int numberOfUnpackableFiles = 0; 
+                    Long combinedUnzippedFileSize = 0L; 
+
+                    /**
+                     * Note that we can't just use zipFile.size(),
+                     * unfortunately, since that's the total number of entries,
+                     * some of which can be directories. So we need to go
+                     * through all the individual zipEntries and count the ones
+                     * that are files.
+                     */
+
+                    for (Enumeration<? extends ZipEntry> entries = zipFile.entries(); entries.hasMoreElements();) {
+                        ZipEntry entry = entries.nextElement();
+                        logger.fine("inside first zip pass; this entry: "+entry.getName());
+                        if (!entry.isDirectory()) {
+                            String shortName = entry.getName().replaceFirst("^.*[\\/]", "");
+                            // ... and, finally, check if it's a "fake" file - a zip archive entry
+                            // created for a MacOS X filesystem element: (these
+                            // start with "._") 
+                            if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
+                                numberOfUnpackableFiles++;
+                                if (numberOfUnpackableFiles > fileNumberLimit) {
+                                    logger.warning("Zip upload - too many files in the zip to process individually.");
+                                    warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
+                                            + "); please upload a zip archive with fewer files, if you want them to be ingested "
+                                            + "as individual DataFiles.";
+                                    throw new IOException();
+                                }
+                                // In addition to counting the files, we can
+                                // also check the file size while we're here, 
+                                // provided the size limit is defined; if a single 
+                                // file is above the individual size limit, unzipped,
+                                // we give up on unpacking this zip archive as well: 
+                                if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) {
+                                    throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit)));
+                                }
+                                // Similarly, we want to check if saving all these unpacked 
+                                // files is going to push the disk usage over the 
+                                // quota:
+                                if (storageQuotaLimit != null) {
+                                    combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize();
+                                    if (combinedUnzippedFileSize > storageQuotaLimit) {
+                                        throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    
+                    // OK we're still here - that means we can proceed unzipping. 
+                    
+                    // Close the ZipFile, re-open as ZipInputStream: 
+                    zipFile.close(); 
+
                     if (charset != null) {
                         unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset);
                     } else {
                         unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));
                     }
 
-                    Long storageQuotaLimitForUnzippedFiles = storageQuotaLimit; 
                     while (true) {
                         try {
                             zipEntry = unZippedIn.getNextEntry();
@@ -304,16 +382,16 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                                     // OK, this seems like an OK file entry - we'll try
                                     // to read it and create a DataFile with it:
 
-                                    File unZippedTempFile = saveInputStreamInTempFile(unZippedIn, fileSizeLimit, storageQuotaLimitForUnzippedFiles);
-                                    DataFile datafile = FileUtil.createSingleDataFile(version, 
-                                            unZippedTempFile, 
-                                            null, 
-                                            shortName,
+                                    String storageIdentifier = FileUtil.generateStorageIdentifier();
+                                    File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier);
+                                    Files.copy(unZippedIn, unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+                                    // No need to check the size of this unpacked file against the size limit, 
+                                    // since we've already checked for that in the first pass.
+                                    
+                                    DataFile datafile = FileUtil.createSingleDataFile(version, null, storageIdentifier, shortName,
                                             MIME_TYPE_UNDETERMINED_DEFAULT,
                                             ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false);
                                     
-                                    storageQuotaLimitForUnzippedFiles = storageQuotaLimitForUnzippedFiles - datafile.getFilesize();
-
                                     if (!fileEntryName.equals(shortName)) {
                                         // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
                                         // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all 
@@ -336,7 +414,9 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                                         String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
 
                                         try {
-                                            recognizedType = determineFileType(new File(tempFileName), shortName);
+                                            recognizedType = determineFileType(unzippedFile, shortName);
+                                            // null the File explicitly, to release any open FDs:
+                                            unzippedFile = null;
                                             logger.fine("File utility recognized unzipped file as " + recognizedType);
                                             if (recognizedType != null && !recognizedType.equals("")) {
                                                 datafile.setContentType(recognizedType);
@@ -373,14 +453,18 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));
                     datafiles.clear();
                 } finally {
+                    if (zipFile != null) {
+                        try {
+                            zipFile.close();
+                        } catch (Exception zEx) {}
+                    }
                     if (unZippedIn != null) {
                         try {
                             unZippedIn.close();
-                        } catch (Exception zEx) {
-                        }
+                        } catch (Exception zEx) {}
                     }
                 }
-                if (datafiles.size() > 0) {
+                if (!datafiles.isEmpty()) {
                     // remove the uploaded zip file:
                     try {
                         Files.delete(tempFile);
@@ -447,7 +531,6 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                         } catch (IOException ioex) {
                             // this one can be ignored
                         }
-
                     }
                 } catch (FileExceedsMaxSizeException | FileExceedsStorageQuotaException femsx) {
                     logger.severe("One of the unzipped shape files exceeded the size limit, or the storage quota; giving up. " + femsx.getMessage());
@@ -468,7 +551,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     logger.warning("Could not remove temp folder, error message : " + ioex.getMessage());
                 }
 
-                if (datafiles.size() > 0) {
+                if (!datafiles.isEmpty()) {
                     // remove the uploaded zip file:
                     try {
                         Files.delete(tempFile);
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 6b1dfe9c163..73d06ad8bee 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -96,20 +96,14 @@
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 
-import org.apache.commons.io.FileUtils;
 
 import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipFile;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
 import org.apache.commons.io.FilenameUtils;
 
 import edu.harvard.iq.dataverse.dataaccess.DataAccessOption;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
-import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
 import java.util.Arrays;
-import java.util.Enumeration;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import ucar.nc2.NetcdfFile;
@@ -848,12 +842,12 @@ public static File saveInputStreamInTempFile(InputStream inputStream, Long fileS
             Long fileSize = tempFile.toFile().length();
             if (fileSizeLimit != null && fileSize > fileSizeLimit) {
                 try {tempFile.toFile().delete();} catch (Exception ex) {}
-                throw new FileExceedsMaxSizeException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)));  
+                throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)));  
             }
             
             if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
                 try {tempFile.toFile().delete();} catch (Exception ex) {}
-                throw new FileExceedsStorageQuotaException (MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)));  
+                throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)));  
             }
             
             return tempFile.toFile();

From f6ea0fd06262a878a4e41048d046bce00d1e9784 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 25 Apr 2023 17:55:58 -0400
Subject: [PATCH 043/922] remaining quota in bytes should never be shown as
 negative, is exceeded. #9361

---
 .../java/edu/harvard/iq/dataverse/DataFileServiceBean.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 5732b9df7ed..2e30e6b76f8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -168,6 +168,9 @@ public void setTotalUsageInBytes(Long totalUsageInBytes) {
         }
         
         public Long getRemainingQuotaInBytes() {
+            if (totalUsageInBytes > totalAllocatedInBytes) {
+                return 0L; 
+            }
             return totalAllocatedInBytes - totalUsageInBytes;
         }
     }

From 30ea450e6c2f32ad3c545249375b510733e8ab13 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 26 Apr 2023 13:14:26 -0400
Subject: [PATCH 044/922] this should resolve the Sword upload errors (wasn't
 calling the new command with the right auth) #9361

---
 .../iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
index bd7067efb7b..c71eeb3d375 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
@@ -309,7 +309,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au
 
             try {
                 //CreateDataFileResult createDataFilesResponse =  FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig);
-                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null);
+                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvReq, editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null);
                 CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                 dataFiles = createDataFilesResult.getDataFiles();
             } catch (CommandException ex) {

From 4955f810437788fef1a345e356308f8ea32c66cb Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 26 Apr 2023 14:38:48 -0400
Subject: [PATCH 045/922] And this should fix the remaining shape file test.
 #9361

---
 .../engine/command/impl/CreateNewDataFilesCommand.java        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 15304502232..365a260cfd2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -522,7 +522,9 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                         if (new_datafile != null) {
                             datafiles.add(new_datafile);
                             // todo: can this new_datafile be null?
-                            storageQuotaLimitForRezippedFiles = storageQuotaLimitForRezippedFiles - new_datafile.getFilesize();
+                            if (storageQuotaLimitForRezippedFiles != null) {
+                                storageQuotaLimitForRezippedFiles = storageQuotaLimitForRezippedFiles - new_datafile.getFilesize();
+                            }
                         } else {
                             logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName());
                         }

From ecac37fbd64c83bfc8d045ae3204ab86dc7bc29d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 2 May 2023 10:52:13 -0400
Subject: [PATCH 046/922] initial Globus Store class with some quick test code

---
 pom.xml                                       |   7 +-
 .../dataaccess/GlobusOverlayAccessIO.java     | 655 ++++++++++++++++++
 .../dataaccess/RemoteOverlayAccessIO.java     |  34 +-
 .../iq/dataverse/settings/JvmSettings.java    |   2 +
 4 files changed, 680 insertions(+), 18 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java

diff --git a/pom.xml b/pom.xml
index 5f514819947..e5b191f0ed7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,8 +167,13 @@
         <dependency>
             <groupId>org.eclipse.microprofile.config</groupId>
             <artifactId>microprofile-config-api</artifactId>
-            <scope>provided</scope>
+            <!--scope>provided</scope-->
         </dependency>
+        <dependency>
+    <groupId>org.apache.geronimo.config</groupId>
+    <artifactId>geronimo-config-impl</artifactId>
+    <version>1.0</version>
+</dependency>
         <dependency>
             <groupId>jakarta.platform</groupId>
             <artifactId>jakarta.jakartaee-api</artifactId>
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
new file mode 100644
index 00000000000..fe62e25ad6f
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -0,0 +1,655 @@
+package edu.harvard.iq.dataverse.dataaccess;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.datavariable.DataVariable;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.util.UrlSignerUtil;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.channels.Channel;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Path;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.util.EntityUtils;
+
+import javax.net.ssl.SSLContext;
+
+/**
+ * @author qqmyers
+ * @param <T> what it stores
+ */
+/*
+ * Globus Overlay Driver
+ * 
+ * StorageIdentifier format:
+ * <globusDriverId>://<local id>//<globusEndpointIdentifier>/<absolutePath>
+ */
+public class GlobusOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
+
+    private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
+
+    private StorageIO<DvObject> baseStore = null;
+    private String path = null;
+    private String endpointWithBasePath = null;
+
+    private static HttpClientContext localContext = HttpClientContext.create();
+    private PoolingHttpClientConnectionManager cm = null;
+    CloseableHttpClient httpclient = null;
+    private int timeout = 1200;
+    private RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
+            .setConnectionRequestTimeout(timeout * 1000).setSocketTimeout(timeout * 1000)
+            .setCookieSpec(CookieSpecs.STANDARD).setExpectContinueEnabled(true).build();
+    private static boolean trustCerts = false;
+    private int httpConcurrency = 4;
+
+    public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
+        super(dvObject, req, driverId);
+        this.setIsLocalFile(false);
+        configureStores(req, driverId, null);
+        logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
+        path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
+        validatePath(path);
+        
+        logger.fine("Relative path: " + path);
+    }
+
+    public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
+        super(null, null, driverId);
+        this.setIsLocalFile(false);
+        configureStores(null, driverId, storageLocation);
+
+        path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
+        validatePath(path);
+        logger.fine("Relative path: " + path);
+    }
+    
+    private void validatePath(String relPath) throws IOException {
+        try {
+            URI absoluteURI = new URI(endpointWithBasePath + "/" + relPath);
+            if(!absoluteURI.normalize().toString().startsWith(endpointWithBasePath)) {
+                throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s endpoint/basePath");
+            }
+        } catch(URISyntaxException use) {
+            throw new IOException("Could not interpret storageidentifier in remote store " + this.driverId);
+        }
+     }
+
+
+    @Override
+    public void open(DataAccessOption... options) throws IOException {
+
+        baseStore.open(options);
+
+        DataAccessRequest req = this.getRequest();
+
+        if (isWriteAccessRequested(options)) {
+            isWriteAccess = true;
+            isReadAccess = false;
+        } else {
+            isWriteAccess = false;
+            isReadAccess = true;
+        }
+
+        if (dvObject instanceof DataFile) {
+            String storageIdentifier = dvObject.getStorageIdentifier();
+
+            DataFile dataFile = this.getDataFile();
+
+            if (req != null && req.getParameter("noVarHeader") != null) {
+                baseStore.setNoVarHeader(true);
+            }
+
+            if (storageIdentifier == null || "".equals(storageIdentifier)) {
+                throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile.");
+            }
+
+            // Fix new DataFiles: DataFiles that have not yet been saved may use this method
+            // when they don't have their storageidentifier in the final form
+            // So we fix it up here. ToDo: refactor so that storageidentifier is generated
+            // by the appropriate StorageIO class and is final from the start.
+            logger.fine("StorageIdentifier is: " + storageIdentifier);
+
+            if (isReadAccess) {
+                if (dataFile.getFilesize() >= 0) {
+                    this.setSize(dataFile.getFilesize());
+                } else {
+                    logger.fine("Setting size");
+                    this.setSize(getSizeFromGlobus());
+                }
+                if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values")
+                        && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
+
+                    List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
+                    String varHeaderLine = generateVariableHeader(datavariables);
+                    this.setVarHeader(varHeaderLine);
+                }
+
+            }
+
+            this.setMimeType(dataFile.getContentType());
+
+            try {
+                this.setFileName(dataFile.getFileMetadata().getLabel());
+            } catch (Exception ex) {
+                this.setFileName("unknown");
+            }
+        } else if (dvObject instanceof Dataset) {
+            throw new IOException(
+                    "Data Access: RemoteOverlay Storage driver does not support dvObject type Dataverse yet");
+        } else if (dvObject instanceof Dataverse) {
+            throw new IOException(
+                    "Data Access: RemoteOverlay Storage driver does not support dvObject type Dataverse yet");
+        } else {
+            this.setSize(getSizeFromGlobus());
+        }
+    }
+
+    private long getSizeFromGlobus() {
+        throw new NotImplementedException();
+        /*
+        long size = -1;
+        HttpHead head = new HttpHead(endpointWithBasePath + "/" + path);
+        try {
+            CloseableHttpResponse response = getSharedHttpClient().execute(head, localContext);
+
+            try {
+                int code = response.getStatusLine().getStatusCode();
+                logger.fine("Response for HEAD: " + code);
+                switch (code) {
+                case 200:
+                    Header[] headers = response.getHeaders(HTTP.CONTENT_LEN);
+                    logger.fine("Num headers: " + headers.length);
+                    String sizeString = response.getHeaders(HTTP.CONTENT_LEN)[0].getValue();
+                    logger.fine("Content-Length: " + sizeString);
+                    size = Long.parseLong(response.getHeaders(HTTP.CONTENT_LEN)[0].getValue());
+                    logger.fine("Found file size: " + size);
+                    break;
+                default:
+                    logger.warning("Response from " + head.getURI().toString() + " was " + code);
+                }
+            } finally {
+                EntityUtils.consume(response.getEntity());
+            }
+        } catch (IOException e) {
+            logger.warning(e.getMessage());
+        }
+        return size;
+        */
+    }
+
+    @Override
+    public InputStream getInputStream() throws IOException {
+        if (super.getInputStream() == null) {
+            try {
+                HttpGet get = new HttpGet(generateTemporaryDownloadUrl(null, null, null));
+                CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
+
+                int code = response.getStatusLine().getStatusCode();
+                switch (code) {
+                case 200:
+                    setInputStream(response.getEntity().getContent());
+                    break;
+                default:
+                    logger.warning("Response from " + get.getURI().toString() + " was " + code);
+                    throw new IOException("Cannot retrieve: " + endpointWithBasePath + "/" + path + " code: " + code);
+                }
+            } catch (Exception e) {
+                logger.warning(e.getMessage());
+                e.printStackTrace();
+                throw new IOException("Error retrieving: " + endpointWithBasePath + "/" + path + " " + e.getMessage());
+
+            }
+            setChannel(Channels.newChannel(super.getInputStream()));
+        }
+        return super.getInputStream();
+    }
+
+    @Override
+    public Channel getChannel() throws IOException {
+        if (super.getChannel() == null) {
+            getInputStream();
+        }
+        return channel;
+    }
+
+    @Override
+    public ReadableByteChannel getReadChannel() throws IOException {
+        // Make sure StorageIO.channel variable exists
+        getChannel();
+        return super.getReadChannel();
+    }
+
+    @Override
+    public void delete() throws IOException {
+        // Delete is best-effort - we tell the remote server and it may or may not
+        // implement this call
+        if (!isDirectAccess()) {
+            throw new IOException("Direct Access IO must be used to permanently delete stored file objects");
+        }
+        try {
+            HttpDelete del = new HttpDelete(endpointWithBasePath + "/" + path);
+            CloseableHttpResponse response = getSharedHttpClient().execute(del, localContext);
+            try {
+                int code = response.getStatusLine().getStatusCode();
+                switch (code) {
+                case 200:
+                    logger.fine("Sent DELETE for " + endpointWithBasePath + "/" + path);
+                default:
+                    logger.fine("Response from DELETE on " + del.getURI().toString() + " was " + code);
+                }
+            } finally {
+                EntityUtils.consume(response.getEntity());
+            }
+        } catch (Exception e) {
+            logger.warning(e.getMessage());
+            throw new IOException("Error deleting: " + endpointWithBasePath + "/" + path);
+
+        }
+
+        // Delete all the cached aux files as well:
+        deleteAllAuxObjects();
+
+    }
+
+    @Override
+    public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException {
+        return baseStore.openAuxChannel(auxItemTag, options);
+    }
+
+    @Override
+    public boolean isAuxObjectCached(String auxItemTag) throws IOException {
+        return baseStore.isAuxObjectCached(auxItemTag);
+    }
+
+    @Override
+    public long getAuxObjectSize(String auxItemTag) throws IOException {
+        return baseStore.getAuxObjectSize(auxItemTag);
+    }
+
+    @Override
+    public Path getAuxObjectAsPath(String auxItemTag) throws IOException {
+        return baseStore.getAuxObjectAsPath(auxItemTag);
+    }
+
+    @Override
+    public void backupAsAux(String auxItemTag) throws IOException {
+        baseStore.backupAsAux(auxItemTag);
+    }
+
+    @Override
+    public void revertBackupAsAux(String auxItemTag) throws IOException {
+        baseStore.revertBackupAsAux(auxItemTag);
+    }
+
+    @Override
+    // this method copies a local filesystem Path into this DataAccess Auxiliary
+    // location:
+    public void savePathAsAux(Path fileSystemPath, String auxItemTag) throws IOException {
+        baseStore.savePathAsAux(fileSystemPath, auxItemTag);
+    }
+
+    @Override
+    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Long filesize) throws IOException {
+        baseStore.saveInputStreamAsAux(inputStream, auxItemTag, filesize);
+    }
+
+    /**
+     * @param inputStream InputStream we want to save
+     * @param auxItemTag  String representing this Auxiliary type ("extension")
+     * @throws IOException if anything goes wrong.
+     */
+    @Override
+    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException {
+        baseStore.saveInputStreamAsAux(inputStream, auxItemTag);
+    }
+
+    @Override
+    public List<String> listAuxObjects() throws IOException {
+        return baseStore.listAuxObjects();
+    }
+
+    @Override
+    public void deleteAuxObject(String auxItemTag) throws IOException {
+        baseStore.deleteAuxObject(auxItemTag);
+    }
+
+    @Override
+    public void deleteAllAuxObjects() throws IOException {
+        baseStore.deleteAllAuxObjects();
+    }
+
+    @Override
+    public String getStorageLocation() throws IOException {
+        String fullStorageLocation = dvObject.getStorageIdentifier();
+        logger.fine("storageidentifier: " + fullStorageLocation);
+        int driverIndex = fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR);
+        if(driverIndex >=0) {
+          fullStorageLocation = fullStorageLocation.substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        }
+        if (this.getDvObject() instanceof Dataset) {
+            throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
+        } else if (this.getDvObject() instanceof DataFile) {
+            fullStorageLocation = StorageIO.getDriverPrefix(this.driverId) + fullStorageLocation;
+        } else if (dvObject instanceof Dataverse) {
+            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
+        }
+        logger.fine("fullStorageLocation: " + fullStorageLocation);
+        return fullStorageLocation;
+    }
+
+    @Override
+    public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
+        throw new UnsupportedDataAccessOperationException(
+                "RemoteOverlayAccessIO: this is a remote DataAccess IO object, it has no local filesystem path associated with it.");
+    }
+
+    @Override
+    public boolean exists() {
+        logger.fine("Exists called");
+        return (getSizeFromGlobus() != -1);
+    }
+
+    @Override
+    public WritableByteChannel getWriteChannel() throws UnsupportedDataAccessOperationException {
+        throw new UnsupportedDataAccessOperationException(
+                "RemoteOverlayAccessIO: there are no write Channels associated with S3 objects.");
+    }
+
+    @Override
+    public OutputStream getOutputStream() throws UnsupportedDataAccessOperationException {
+        throw new UnsupportedDataAccessOperationException(
+                "RemoteOverlayAccessIO: there are no output Streams associated with S3 objects.");
+    }
+
+    @Override
+    public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException {
+        return baseStore.getAuxFileAsInputStream(auxItemTag);
+    }
+
+    @Override
+    public boolean downloadRedirectEnabled() {
+        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".download-redirect");
+        if ("true".equalsIgnoreCase(optionValue)) {
+            return true;
+        }
+        return false;
+    }
+    
+    public boolean downloadRedirectEnabled(String auxObjectTag) {
+        return baseStore.downloadRedirectEnabled(auxObjectTag);
+    }
+
+    @Override
+    public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName)
+            throws IOException {
+
+        // ToDo - support remote auxiliary Files
+        if (auxiliaryTag == null) {
+            String secretKey = System.getProperty("dataverse.files." + this.driverId + ".secret-key");
+            if (secretKey == null) {
+                return endpointWithBasePath + "/" + path;
+            } else {
+                return UrlSignerUtil.signUrl(endpointWithBasePath + "/" + path, getUrlExpirationMinutes(), null, "GET",
+                        secretKey);
+            }
+        } else {
+            return baseStore.generateTemporaryDownloadUrl(auxiliaryTag, auxiliaryType, auxiliaryFileName);
+        }
+    }
+
+    int getUrlExpirationMinutes() {
+        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".url-expiration-minutes");
+        if (optionValue != null) {
+            Integer num;
+            try {
+                num = Integer.parseInt(optionValue);
+            } catch (NumberFormatException ex) {
+                num = null;
+            }
+            if (num != null) {
+                return num;
+            }
+        }
+        return 60;
+    }
+
+    private void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
+        endpointWithBasePath = JvmSettings.BASE_URI.lookup(this.driverId);
+        logger.info("base-uri is " + endpointWithBasePath);
+        if (endpointWithBasePath == null) {
+            throw new IOException("dataverse.files." + this.driverId + ".base-uri is required");
+        } else {
+            try {
+                new URI(endpointWithBasePath);
+            } catch (Exception e) {
+                logger.warning(
+                        "Trouble interpreting base-url for store: " + this.driverId + " : " + e.getLocalizedMessage());
+                throw new IOException("Can't interpret base-url as a URI");
+            }
+
+        }
+
+        if (baseStore == null) {
+            String baseDriverId = getBaseStoreIdFor(driverId);
+            String fullStorageLocation = null;
+            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+            
+            if(dvObject  instanceof Dataset) {
+                baseStore = DataAccess.getStorageIO(dvObject, req, baseDriverId);
+            } else {
+                if (this.getDvObject() != null) {
+                    fullStorageLocation = getStoragePath();
+
+                    // S3 expects <id>://<bucketname>/<key>
+                    switch (baseDriverType) {
+                    case DataAccess.S3:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + System.getProperty("dataverse.files." + baseDriverId + ".bucket-name") + "/"
+                                + fullStorageLocation;
+                        break;
+                    case DataAccess.FILE:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files") + "/"
+                                + fullStorageLocation;
+                        break;
+                    default:
+                        logger.warning("Not Implemented: RemoteOverlay store with base store type: "
+                                + System.getProperty("dataverse.files." + baseDriverId + ".type"));
+                        throw new IOException("Not implemented");
+                    }
+
+                } else if (storageLocation != null) {
+                    // <remoteDriverId>://<baseStorageIdentifier>//<baseUrlPath>
+                    //remoteDriverId:// is removed if coming through directStorageIO
+                    int index = storageLocation.indexOf(DataAccess.SEPARATOR);
+                    if(index > 0) {
+                        storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
+                    }
+                    //THe base store needs the baseStoreIdentifier and not the relative URL
+                    fullStorageLocation = storageLocation.substring(0, storageLocation.indexOf("//"));
+
+                    switch (baseDriverType) {
+                    case DataAccess.S3:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + System.getProperty("dataverse.files." + baseDriverId + ".bucket-name") + "/"
+                                + fullStorageLocation;
+                        break;
+                    case DataAccess.FILE:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files") + "/"
+                                + fullStorageLocation;
+                        break;
+                    default:
+                        logger.warning("Not Implemented: RemoteOverlay store with base store type: "
+                                + System.getProperty("dataverse.files." + baseDriverId + ".type"));
+                        throw new IOException("Not implemented");
+                    }
+                }
+                baseStore = DataAccess.getDirectStorageIO(fullStorageLocation);
+            }
+            if (baseDriverType.contentEquals(DataAccess.S3)) {
+                ((S3AccessIO<?>) baseStore).setMainDriver(false);
+            }
+        }
+        remoteStoreName = System.getProperty("dataverse.files." + this.driverId + ".remote-store-name");
+        try {
+          remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
+        } catch(MalformedURLException mfue) {
+            logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
+        }
+    }
+
+    //Convenience method to assemble the path, starting with the DOI authority/identifier/, that is needed to create a base store via DataAccess.getDirectStorageIO - the caller has to add the store type specific prefix required.
+    private String getStoragePath() throws IOException {
+        String fullStoragePath = dvObject.getStorageIdentifier();
+        logger.fine("storageidentifier: " + fullStoragePath);
+        int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
+        if(driverIndex >=0) {
+          fullStoragePath = fullStoragePath.substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        }
+        int suffixIndex = fullStoragePath.indexOf("//");
+        if(suffixIndex >=0) {
+          fullStoragePath = fullStoragePath.substring(0, suffixIndex);
+        }
+        if (this.getDvObject() instanceof Dataset) {
+            fullStoragePath = this.getDataset().getAuthorityForFileStorage() + "/"
+                    + this.getDataset().getIdentifierForFileStorage() + "/" + fullStoragePath;
+        } else if (this.getDvObject() instanceof DataFile) {
+            fullStoragePath = this.getDataFile().getOwner().getAuthorityForFileStorage() + "/"
+                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath; 
+        }else if (dvObject instanceof Dataverse) {
+            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
+        }
+        logger.fine("fullStoragePath: " + fullStoragePath);
+        return fullStoragePath;
+    }
+    
+    public CloseableHttpClient getSharedHttpClient() {
+        if (httpclient == null) {
+            try {
+                initHttpPool();
+                httpclient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(config).build();
+
+            } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException ex) {
+                logger.warning(ex.getMessage());
+            }
+        }
+        return httpclient;
+    }
+
+    private void initHttpPool() throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException {
+        if (trustCerts) {
+            // use the TrustSelfSignedStrategy to allow Self Signed Certificates
+            SSLContext sslContext;
+            SSLConnectionSocketFactory connectionFactory;
+
+            sslContext = SSLContextBuilder.create().loadTrustMaterial(new TrustAllStrategy()).build();
+            // create an SSL Socket Factory to use the SSLContext with the trust self signed
+            // certificate strategy
+            // and allow all hosts verifier.
+            connectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
+
+            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
+                    .register("https", connectionFactory).build();
+            cm = new PoolingHttpClientConnectionManager(registry);
+        } else {
+            cm = new PoolingHttpClientConnectionManager();
+        }
+        cm.setDefaultMaxPerRoute(httpConcurrency);
+        cm.setMaxTotal(httpConcurrency > 20 ? httpConcurrency : 20);
+    }
+
+    @Override
+    public void savePath(Path fileSystemPath) throws IOException {
+        throw new UnsupportedDataAccessOperationException(
+                "RemoteOverlayAccessIO: savePath() not implemented in this storage driver.");
+
+    }
+
+    @Override
+    public void saveInputStream(InputStream inputStream) throws IOException {
+        throw new UnsupportedDataAccessOperationException(
+                "RemoteOverlayAccessIO: saveInputStream() not implemented in this storage driver.");
+
+    }
+
+    @Override
+    public void saveInputStream(InputStream inputStream, Long filesize) throws IOException {
+        throw new UnsupportedDataAccessOperationException(
+                "RemoteOverlayAccessIO: saveInputStream(InputStream, Long) not implemented in this storage driver.");
+
+    }
+
+    protected static boolean isValidIdentifier(String driverId, String storageId) {
+        String urlPath = storageId.substring(storageId.lastIndexOf("//") + 2);
+        String baseUri = System.getProperty("dataverse.files." + driverId + ".base-uri");
+        try {
+            URI absoluteURI = new URI(baseUri + "/" + urlPath);
+            if(!absoluteURI.normalize().toString().startsWith(baseUri)) {
+                logger.warning("storageidentifier doesn't start with " + driverId + "'s base-url: " + storageId);
+                return false;
+            }
+        } catch(URISyntaxException use) {
+            logger.warning("Could not interpret storageidentifier in remote store " + driverId + " : " + storageId);
+            logger.warning(use.getLocalizedMessage());
+            return false;
+        }
+        return true;
+    }
+
+    public static String getBaseStoreIdFor(String driverId) {
+        return System.getProperty("dataverse.files." + driverId + ".base-store");
+    }
+
+    @Override
+    public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOException {
+        return baseStore.cleanUp(filter, dryRun);
+    }
+    
+    public static void main(String[] args) {
+        System.out.println("Running the main method");
+        if (args.length > 0) {
+            System.out.printf("List of arguments: {}", Arrays.toString(args));
+        }
+        System.setProperty("dataverse.files.globus.base-uri", "12345/top");
+        System.out.println("Valid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
+        logger.info(JvmSettings.BASE_URI.lookup("globus"));
+    }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index 66c6a4cc2ee..ee2b6779cba 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -65,7 +65,7 @@ public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
 
     private StorageIO<DvObject> baseStore = null;
-    private String urlPath = null;
+    private String path = null;
     private String baseUrl = null;
 
     private static HttpClientContext localContext = HttpClientContext.create();
@@ -83,10 +83,10 @@ public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId)
         this.setIsLocalFile(false);
         configureStores(req, driverId, null);
         logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
-        urlPath = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
-        validatePath(urlPath);
+        path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
+        validatePath(path);
         
-        logger.fine("Base URL: " + urlPath);
+        logger.fine("Base URL: " + path);
     }
 
     public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOException {
@@ -94,14 +94,14 @@ public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOE
         this.setIsLocalFile(false);
         configureStores(null, driverId, storageLocation);
 
-        urlPath = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
-        validatePath(urlPath);
-        logger.fine("Base URL: " + urlPath);
+        path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
+        validatePath(path);
+        logger.fine("Base URL: " + path);
     }
     
-    private void validatePath(String path) throws IOException {
+    private void validatePath(String relPath) throws IOException {
         try {
-            URI absoluteURI = new URI(baseUrl + "/" + urlPath);
+            URI absoluteURI = new URI(baseUrl + "/" + relPath);
             if(!absoluteURI.normalize().toString().startsWith(baseUrl)) {
                 throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s base-url");
             }
@@ -182,7 +182,7 @@ public void open(DataAccessOption... options) throws IOException {
 
     private long getSizeFromHttpHeader() {
         long size = -1;
-        HttpHead head = new HttpHead(baseUrl + "/" + urlPath);
+        HttpHead head = new HttpHead(baseUrl + "/" + path);
         try {
             CloseableHttpResponse response = getSharedHttpClient().execute(head, localContext);
 
@@ -224,12 +224,12 @@ public InputStream getInputStream() throws IOException {
                     break;
                 default:
                     logger.warning("Response from " + get.getURI().toString() + " was " + code);
-                    throw new IOException("Cannot retrieve: " + baseUrl + "/" + urlPath + " code: " + code);
+                    throw new IOException("Cannot retrieve: " + baseUrl + "/" + path + " code: " + code);
                 }
             } catch (Exception e) {
                 logger.warning(e.getMessage());
                 e.printStackTrace();
-                throw new IOException("Error retrieving: " + baseUrl + "/" + urlPath + " " + e.getMessage());
+                throw new IOException("Error retrieving: " + baseUrl + "/" + path + " " + e.getMessage());
 
             }
             setChannel(Channels.newChannel(super.getInputStream()));
@@ -260,13 +260,13 @@ public void delete() throws IOException {
             throw new IOException("Direct Access IO must be used to permanently delete stored file objects");
         }
         try {
-            HttpDelete del = new HttpDelete(baseUrl + "/" + urlPath);
+            HttpDelete del = new HttpDelete(baseUrl + "/" + path);
             CloseableHttpResponse response = getSharedHttpClient().execute(del, localContext);
             try {
                 int code = response.getStatusLine().getStatusCode();
                 switch (code) {
                 case 200:
-                    logger.fine("Sent DELETE for " + baseUrl + "/" + urlPath);
+                    logger.fine("Sent DELETE for " + baseUrl + "/" + path);
                 default:
                     logger.fine("Response from DELETE on " + del.getURI().toString() + " was " + code);
                 }
@@ -275,7 +275,7 @@ public void delete() throws IOException {
             }
         } catch (Exception e) {
             logger.warning(e.getMessage());
-            throw new IOException("Error deleting: " + baseUrl + "/" + urlPath);
+            throw new IOException("Error deleting: " + baseUrl + "/" + path);
 
         }
 
@@ -420,9 +420,9 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
         if (auxiliaryTag == null) {
             String secretKey = System.getProperty("dataverse.files." + this.driverId + ".secret-key");
             if (secretKey == null) {
-                return baseUrl + "/" + urlPath;
+                return baseUrl + "/" + path;
             } else {
-                return UrlSignerUtil.signUrl(baseUrl + "/" + urlPath, getUrlExpirationMinutes(), null, "GET",
+                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET",
                         secretKey);
             }
         } else {
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index 86130f5146e..4fb895f5adc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -47,6 +47,8 @@ public enum JvmSettings {
     // FILES SETTINGS
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
+    FILES(SCOPE_FILES),
+    BASE_URI(FILES, "base-uri"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),

From 2c4c927cc8f20d53ee1aaaf1979b793ee53f9b3f Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 5 May 2023 14:13:02 -0400
Subject: [PATCH 047/922] add token

---
 .../dataaccess/GlobusOverlayAccessIO.java     | 171 +++++++++++-------
 .../iq/dataverse/settings/JvmSettings.java    |   1 +
 2 files changed, 111 insertions(+), 61 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index fe62e25ad6f..050b9ddc176 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -7,6 +7,7 @@
 import edu.harvard.iq.dataverse.datavariable.DataVariable;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -31,6 +32,7 @@
 import java.util.logging.Logger;
 
 import org.apache.commons.lang3.NotImplementedException;
+import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -49,6 +51,7 @@
 import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.util.EntityUtils;
 
+import javax.json.JsonObject;
 import javax.net.ssl.SSLContext;
 
 /**
@@ -58,8 +61,8 @@
 /*
  * Globus Overlay Driver
  * 
- * StorageIdentifier format:
- * <globusDriverId>://<local id>//<globusEndpointIdentifier>/<absolutePath>
+ * StorageIdentifier format: <globusDriverId>://<local
+ * id>//<globusEndpointIdentifier>/<absolutePath>
  */
 public class GlobusOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
@@ -68,6 +71,7 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
     private StorageIO<DvObject> baseStore = null;
     private String path = null;
     private String endpointWithBasePath = null;
+    private String globusToken = null;
 
     private static HttpClientContext localContext = HttpClientContext.create();
     private PoolingHttpClientConnectionManager cm = null;
@@ -86,7 +90,7 @@ public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId)
         logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
         path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
         validatePath(path);
-        
+
         logger.fine("Relative path: " + path);
     }
 
@@ -99,18 +103,17 @@ public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOE
         validatePath(path);
         logger.fine("Relative path: " + path);
     }
-    
+
     private void validatePath(String relPath) throws IOException {
         try {
             URI absoluteURI = new URI(endpointWithBasePath + "/" + relPath);
-            if(!absoluteURI.normalize().toString().startsWith(endpointWithBasePath)) {
+            if (!absoluteURI.normalize().toString().startsWith(endpointWithBasePath)) {
                 throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s endpoint/basePath");
             }
-        } catch(URISyntaxException use) {
+        } catch (URISyntaxException use) {
             throw new IOException("Could not interpret storageidentifier in remote store " + this.driverId);
         }
-     }
-
+    }
 
     @Override
     public void open(DataAccessOption... options) throws IOException {
@@ -181,37 +184,64 @@ public void open(DataAccessOption... options) throws IOException {
         }
     }
 
+    // Call the Globus API to get the file size
     private long getSizeFromGlobus() {
-        throw new NotImplementedException();
-        /*
-        long size = -1;
-        HttpHead head = new HttpHead(endpointWithBasePath + "/" + path);
+        // Construct Globus URL
+        URI absoluteURI = null;
         try {
-            CloseableHttpResponse response = getSharedHttpClient().execute(head, localContext);
-
-            try {
-                int code = response.getStatusLine().getStatusCode();
-                logger.fine("Response for HEAD: " + code);
-                switch (code) {
-                case 200:
-                    Header[] headers = response.getHeaders(HTTP.CONTENT_LEN);
-                    logger.fine("Num headers: " + headers.length);
-                    String sizeString = response.getHeaders(HTTP.CONTENT_LEN)[0].getValue();
-                    logger.fine("Content-Length: " + sizeString);
-                    size = Long.parseLong(response.getHeaders(HTTP.CONTENT_LEN)[0].getValue());
-                    logger.fine("Found file size: " + size);
-                    break;
-                default:
-                    logger.warning("Response from " + head.getURI().toString() + " was " + code);
-                }
-            } finally {
-                EntityUtils.consume(response.getEntity());
+            int filenameStart = path.lastIndexOf("/") + 1;
+            int pathStart = endpointWithBasePath.indexOf("/") + 1;
+
+            String directoryPath = (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "")
+                    + path.substring(0, filenameStart);
+            String filename = path.substring(filenameStart);
+            String endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart - 1) : endpointWithBasePath;
+
+            absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint + "/ls?path=" + path + "&filter=name:" + filename);
+            HttpGet get = new HttpGet(absoluteURI);
+            String token = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
+            logger.info("Token is " + token);
+            get.addHeader("Authorization", "Bearer " + token);
+            CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                //Get reponse as string
+                String responseString = EntityUtils.toString(response.getEntity());
+                logger.fine("Response from " + get.getURI().toString() + " is: " + responseString);
+                JsonObject responseJson = JsonUtil.getJsonObject(responseString);
+                return (long) responseJson.getInt("size");
+            } else {
+                logger.warning("Response from " + get.getURI().toString() + " was " + response.getStatusLine().getStatusCode());
+                logger.info(EntityUtils.toString(response.getEntity()));
             }
+        } catch (URISyntaxException e) {
+            // Should have been caught in validatePath
+            e.printStackTrace();
+        } catch (ClientProtocolException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
         } catch (IOException e) {
-            logger.warning(e.getMessage());
+            // TODO Auto-generated catch block
+            e.printStackTrace();
         }
-        return size;
-        */
+        return -1;
+
+        /*
+         * long size = -1; HttpHead head = new HttpHead(endpointWithBasePath + "/" +
+         * path); try { CloseableHttpResponse response =
+         * getSharedHttpClient().execute(head, localContext);
+         * 
+         * try { int code = response.getStatusLine().getStatusCode();
+         * logger.fine("Response for HEAD: " + code); switch (code) { case 200: Header[]
+         * headers = response.getHeaders(HTTP.CONTENT_LEN); logger.fine("Num headers: "
+         * + headers.length); String sizeString =
+         * response.getHeaders(HTTP.CONTENT_LEN)[0].getValue();
+         * logger.fine("Content-Length: " + sizeString); size =
+         * Long.parseLong(response.getHeaders(HTTP.CONTENT_LEN)[0].getValue());
+         * logger.fine("Found file size: " + size); break; default:
+         * logger.warning("Response from " + head.getURI().toString() + " was " + code);
+         * } } finally { EntityUtils.consume(response.getEntity()); } } catch
+         * (IOException e) { logger.warning(e.getMessage()); } return size;
+         */
     }
 
     @Override
@@ -360,8 +390,9 @@ public String getStorageLocation() throws IOException {
         String fullStorageLocation = dvObject.getStorageIdentifier();
         logger.fine("storageidentifier: " + fullStorageLocation);
         int driverIndex = fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR);
-        if(driverIndex >=0) {
-          fullStorageLocation = fullStorageLocation.substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        if (driverIndex >= 0) {
+            fullStorageLocation = fullStorageLocation
+                    .substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
         }
         if (this.getDvObject() instanceof Dataset) {
             throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
@@ -411,7 +442,7 @@ public boolean downloadRedirectEnabled() {
         }
         return false;
     }
-    
+
     public boolean downloadRedirectEnabled(String auxObjectTag) {
         return baseStore.downloadRedirectEnabled(auxObjectTag);
     }
@@ -469,9 +500,10 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
         if (baseStore == null) {
             String baseDriverId = getBaseStoreIdFor(driverId);
             String fullStorageLocation = null;
-            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
-            
-            if(dvObject  instanceof Dataset) {
+            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type",
+                    DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+
+            if (dvObject instanceof Dataset) {
                 baseStore = DataAccess.getStorageIO(dvObject, req, baseDriverId);
             } else {
                 if (this.getDvObject() != null) {
@@ -486,8 +518,8 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                         break;
                     case DataAccess.FILE:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files") + "/"
-                                + fullStorageLocation;
+                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
+                                + "/" + fullStorageLocation;
                         break;
                     default:
                         logger.warning("Not Implemented: RemoteOverlay store with base store type: "
@@ -497,12 +529,12 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
 
                 } else if (storageLocation != null) {
                     // <remoteDriverId>://<baseStorageIdentifier>//<baseUrlPath>
-                    //remoteDriverId:// is removed if coming through directStorageIO
+                    // remoteDriverId:// is removed if coming through directStorageIO
                     int index = storageLocation.indexOf(DataAccess.SEPARATOR);
-                    if(index > 0) {
+                    if (index > 0) {
                         storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
                     }
-                    //THe base store needs the baseStoreIdentifier and not the relative URL
+                    // THe base store needs the baseStoreIdentifier and not the relative URL
                     fullStorageLocation = storageLocation.substring(0, storageLocation.indexOf("//"));
 
                     switch (baseDriverType) {
@@ -513,8 +545,8 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                         break;
                     case DataAccess.FILE:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files") + "/"
-                                + fullStorageLocation;
+                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
+                                + "/" + fullStorageLocation;
                         break;
                     default:
                         logger.warning("Not Implemented: RemoteOverlay store with base store type: "
@@ -530,37 +562,41 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
         }
         remoteStoreName = System.getProperty("dataverse.files." + this.driverId + ".remote-store-name");
         try {
-          remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
-        } catch(MalformedURLException mfue) {
+            remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
+        } catch (MalformedURLException mfue) {
             logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
         }
     }
 
-    //Convenience method to assemble the path, starting with the DOI authority/identifier/, that is needed to create a base store via DataAccess.getDirectStorageIO - the caller has to add the store type specific prefix required.
+    // Convenience method to assemble the path, starting with the DOI
+    // authority/identifier/, that is needed to create a base store via
+    // DataAccess.getDirectStorageIO - the caller has to add the store type specific
+    // prefix required.
     private String getStoragePath() throws IOException {
         String fullStoragePath = dvObject.getStorageIdentifier();
         logger.fine("storageidentifier: " + fullStoragePath);
         int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
-        if(driverIndex >=0) {
-          fullStoragePath = fullStoragePath.substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        if (driverIndex >= 0) {
+            fullStoragePath = fullStoragePath
+                    .substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
         }
         int suffixIndex = fullStoragePath.indexOf("//");
-        if(suffixIndex >=0) {
-          fullStoragePath = fullStoragePath.substring(0, suffixIndex);
+        if (suffixIndex >= 0) {
+            fullStoragePath = fullStoragePath.substring(0, suffixIndex);
         }
         if (this.getDvObject() instanceof Dataset) {
             fullStoragePath = this.getDataset().getAuthorityForFileStorage() + "/"
                     + this.getDataset().getIdentifierForFileStorage() + "/" + fullStoragePath;
         } else if (this.getDvObject() instanceof DataFile) {
             fullStoragePath = this.getDataFile().getOwner().getAuthorityForFileStorage() + "/"
-                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath; 
-        }else if (dvObject instanceof Dataverse) {
+                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath;
+        } else if (dvObject instanceof Dataverse) {
             throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
         }
         logger.fine("fullStoragePath: " + fullStoragePath);
         return fullStoragePath;
     }
-    
+
     public CloseableHttpClient getSharedHttpClient() {
         if (httpclient == null) {
             try {
@@ -622,11 +658,11 @@ protected static boolean isValidIdentifier(String driverId, String storageId) {
         String baseUri = System.getProperty("dataverse.files." + driverId + ".base-uri");
         try {
             URI absoluteURI = new URI(baseUri + "/" + urlPath);
-            if(!absoluteURI.normalize().toString().startsWith(baseUri)) {
+            if (!absoluteURI.normalize().toString().startsWith(baseUri)) {
                 logger.warning("storageidentifier doesn't start with " + driverId + "'s base-url: " + storageId);
                 return false;
             }
-        } catch(URISyntaxException use) {
+        } catch (URISyntaxException use) {
             logger.warning("Could not interpret storageidentifier in remote store " + driverId + " : " + storageId);
             logger.warning(use.getLocalizedMessage());
             return false;
@@ -642,14 +678,27 @@ public static String getBaseStoreIdFor(String driverId) {
     public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOException {
         return baseStore.cleanUp(filter, dryRun);
     }
-    
+
     public static void main(String[] args) {
         System.out.println("Running the main method");
         if (args.length > 0) {
             System.out.printf("List of arguments: {}", Arrays.toString(args));
         }
-        System.setProperty("dataverse.files.globus.base-uri", "12345/top");
+        System.setProperty("dataverse.files.globus.base-uri", "2791b83e-b989-47c5-a7fa-ce65fd949522");
         System.out.println("Valid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
+        System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOlprRmxGejNTWDlkTVpUNk92ZmVJaFQyTWY0SDd4cXBoTDNSS29vUmRGVlE9");
+        System.setProperty("dataverse.files.globus.base-store","file");
+        System.setProperty("dataverse.files.file.type",
+                DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+        System.setProperty("dataverse.files.file.directory", "/tmp/files");
         logger.info(JvmSettings.BASE_URI.lookup("globus"));
+        try {
+            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>("globus://1234//2791b83e-b989-47c5-a7fa-ce65fd949522/hdc1/image001.mrc", "globus");
+        logger.info("Size is " + gsio.getSizeFromGlobus());
+        
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index 4fb895f5adc..eac8411c939 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -49,6 +49,7 @@ public enum JvmSettings {
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
     FILES(SCOPE_FILES),
     BASE_URI(FILES, "base-uri"),
+    GLOBUS_TOKEN(FILES, "globus-token"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),

From cef601ee6af94fcfa56d9d02efb807d2c6bb20d2 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 9 May 2023 13:33:17 -0400
Subject: [PATCH 048/922] A new, experimental version of the Add Files command
 with the dynamic permissions lookup that allows it to be checked on either
 the Dataset or the parent Collection (when files are bing added in the
 context of creating a new dataset via the Add Dataset page). #9361

---
 .../impl/CreateNewDataFilesCommand.java       | 49 ++++++++++++++++++-
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 365a260cfd2..fb4f7a7db86 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -9,11 +9,12 @@
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
-import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+//import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
 import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
+import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -42,8 +43,12 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.logging.Logger;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipFile;
@@ -57,7 +62,10 @@
  *
  * @author landreev
  */
-@RequiredPermissions( Permission.EditDataset )
+// Note the commented out @RequiredPermissions. We need to use dynamic 
+// permissions instead, to accommodate both adding files to an existing 
+// dataset and files being uploaded on create of a new dataset. 
+//@RequiredPermissions( Permission.EditDataset )
 public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileResult> {
     private static final Logger logger = Logger.getLogger(CreateNewDataFilesCommand.class.getCanonicalName());
     
@@ -68,6 +76,7 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private final String newStorageIdentifier; 
     private final String newCheckSum; 
     private DataFile.ChecksumType newCheckSumType;
+    private final Dataverse dataverse;
 
     public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum) {
         this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, null);
@@ -83,7 +92,25 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
         this.newStorageIdentifier = newStorageIdentifier; 
         this.newCheckSum = newCheckSum; 
         this.newCheckSumType = newCheckSumType;
+        this.dataverse = null; 
     }
+    
+    // This version of the command must be used when files are created in the 
+    // context of creating a brand new dataset (from the Add Dataset page):
+    
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, DataFile.ChecksumType newCheckSumType, Dataverse dataverse) {
+        super(aRequest, dataverse);
+        
+        this.version = version;
+        this.inputStream = inputStream;
+        this.fileName = fileName;
+        this.suppliedContentType = suppliedContentType; 
+        this.newStorageIdentifier = newStorageIdentifier; 
+        this.newCheckSum = newCheckSum; 
+        this.newCheckSumType = newCheckSumType;
+        this.dataverse = dataverse;
+    }
+    
 
     @Override
     public CreateDataFileResult execute(CommandContext ctxt) throws CommandException {
@@ -637,4 +664,22 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
 
         return CreateDataFileResult.error(fileName, finalType);
     }   // end createDataFiles
+    
+    @Override
+    public Map<String, Set<Permission>> getRequiredPermissions() {
+        Map<String, Set<Permission>> ret = new HashMap<>();
+
+        ret.put("", new HashSet<>());
+        
+        if (dataverse != null) {
+            // The command is called in the context of uploading files on 
+            // create of a new dataset
+            ret.get("").add(Permission.AddDataset);
+        } else {
+            // An existing dataset
+            ret.get("").add(Permission.EditDataset);
+        }
+
+        return ret;
+    }
 }

From 3c3378f5a3bf39eff13a582d0dc52a2a5549af8f Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 9 May 2023 14:53:25 -0400
Subject: [PATCH 049/922] start refactoring Globus bean

---
 .../dataaccess/GlobusOverlayAccessIO.java     | 28 +++++++++-----
 .../iq/dataverse/globus/AccessToken.java      |  2 +-
 .../dataverse/globus/GlobusServiceBean.java   | 37 +++++++++++--------
 3 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 050b9ddc176..0d7c5458e14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -5,6 +5,8 @@
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.datavariable.DataVariable;
+import edu.harvard.iq.dataverse.globus.AccessToken;
+import edu.harvard.iq.dataverse.globus.GlobusServiceBean;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
@@ -28,10 +30,8 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Predicate;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.commons.lang3.NotImplementedException;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
@@ -83,6 +83,8 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
     private static boolean trustCerts = false;
     private int httpConcurrency = 4;
 
+    private String globusAccessToken = null;
+
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
         this.setIsLocalFile(false);
@@ -190,18 +192,19 @@ private long getSizeFromGlobus() {
         URI absoluteURI = null;
         try {
             int filenameStart = path.lastIndexOf("/") + 1;
-            int pathStart = endpointWithBasePath.indexOf("/") + 1;
-
-            String directoryPath = (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "")
+            int pathStart = endpointWithBasePath.indexOf("/");
+logger.info("endpointWithBasePath: " + endpointWithBasePath);
+            String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "")
                     + path.substring(0, filenameStart);
+            logger.info("directoryPath: " + directoryPath);
             String filename = path.substring(filenameStart);
             String endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart - 1) : endpointWithBasePath;
 
-            absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint + "/ls?path=" + path + "&filter=name:" + filename);
+            absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint + "/ls?path=" + directoryPath + "&filter=name:" + filename);
             HttpGet get = new HttpGet(absoluteURI);
-            String token = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
-            logger.info("Token is " + token);
-            get.addHeader("Authorization", "Bearer " + token);
+            
+            logger.info("Token is " + globusAccessToken);
+            get.addHeader("Authorization", "Bearer " + globusAccessToken);
             CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
             if (response.getStatusLine().getStatusCode() == 200) {
                 //Get reponse as string
@@ -482,6 +485,8 @@ int getUrlExpirationMinutes() {
     }
 
     private void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
+        AccessToken accessToken = GlobusServiceBean.getClientToken(JvmSettings.GLOBUS_TOKEN.lookup(driverId));
+        globusAccessToken = accessToken.getOtherTokens().get(0).getAccessToken();
         endpointWithBasePath = JvmSettings.BASE_URI.lookup(this.driverId);
         logger.info("base-uri is " + endpointWithBasePath);
         if (endpointWithBasePath == null) {
@@ -692,8 +697,11 @@ public static void main(String[] args) {
                 DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
         System.setProperty("dataverse.files.file.directory", "/tmp/files");
         logger.info(JvmSettings.BASE_URI.lookup("globus"));
+        
+        
+        
         try {
-            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>("globus://1234//2791b83e-b989-47c5-a7fa-ce65fd949522/hdc1/image001.mrc", "globus");
+            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>("globus://1234///hdc1/image001.mrc", "globus");
         logger.info("Size is " + gsio.getSizeFromGlobus());
         
         } catch (IOException e) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/AccessToken.java b/src/main/java/edu/harvard/iq/dataverse/globus/AccessToken.java
index 877fc68e4a1..c93e2c6aa94 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/AccessToken.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/AccessToken.java
@@ -46,7 +46,7 @@ String getRefreshToken() {
         return refreshToken;
     }
 
-    ArrayList<AccessToken> getOtherTokens() {
+    public ArrayList<AccessToken> getOtherTokens() {
         return otherTokens;
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 9d80c5cc280..c2137dd1f47 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -167,7 +167,8 @@ public void updatePermision(AccessToken clientTokenUser, String directory, Strin
     public void deletePermision(String ruleId, Logger globusLogger) throws MalformedURLException {
 
         if (ruleId.length() > 0) {
-            AccessToken clientTokenUser = getClientToken();
+            AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
+
             globusLogger.info("Start deleting permissions.");
             String globusEndpoint = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusEndpoint, "");
 
@@ -264,15 +265,21 @@ public GlobusTask getTask(AccessToken clientTokenUser, String taskId, Logger glo
         return task;
     }
 
-    public AccessToken getClientToken() throws MalformedURLException {
-        String globusBasicToken = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, "");
-        URL url = new URL(
-                "https://auth.globus.org/v2/oauth2/token?scope=openid+email+profile+urn:globus:auth:scope:transfer.api.globus.org:all&grant_type=client_credentials");
-
-        MakeRequestResponse result = makeRequest(url, "Basic", globusBasicToken, "POST", null);
+    public static AccessToken getClientToken(String globusBasicToken) {
+        URL url;
         AccessToken clientTokenUser = null;
-        if (result.status == 200) {
-            clientTokenUser = parseJson(result.jsonResponse, AccessToken.class, true);
+
+        try {
+            url = new URL(
+                    "https://auth.globus.org/v2/oauth2/token?scope=openid+email+profile+urn:globus:auth:scope:transfer.api.globus.org:all&grant_type=client_credentials");
+
+            MakeRequestResponse result = makeRequest(url, "Basic", globusBasicToken, "POST", null);
+            if (result.status == 200) {
+                clientTokenUser = parseJson(result.jsonResponse, AccessToken.class, true);
+            }
+        } catch (MalformedURLException e) {
+            // On a statically defined URL...
+            e.printStackTrace();
         }
         return clientTokenUser;
     }
@@ -306,7 +313,7 @@ public AccessToken getAccessToken(HttpServletRequest origRequest, String globusB
 
     }
 
-    public MakeRequestResponse makeRequest(URL url, String authType, String authCode, String method,
+    public static MakeRequestResponse makeRequest(URL url, String authType, String authCode, String method,
             String jsonString) {
         String str = null;
         HttpURLConnection connection = null;
@@ -359,7 +366,7 @@ public MakeRequestResponse makeRequest(URL url, String authType, String authCode
 
     }
 
-    private StringBuilder readResultJson(InputStream in) {
+    private static StringBuilder readResultJson(InputStream in) {
         StringBuilder sb = null;
         try {
 
@@ -378,7 +385,7 @@ private StringBuilder readResultJson(InputStream in) {
         return sb;
     }
 
-    private <T> T parseJson(String sb, Class<T> jsonParserClass, boolean namingPolicy) {
+    private static <T> T parseJson(String sb, Class<T> jsonParserClass, boolean namingPolicy) {
         if (sb != null) {
             Gson gson = null;
             if (namingPolicy) {
@@ -420,7 +427,7 @@ public String getDirectory(String datasetId) {
 
     }
 
-    class MakeRequestResponse {
+    static class MakeRequestResponse {
         public String jsonResponse;
         public int status;
 
@@ -451,7 +458,7 @@ public boolean giveGlobusPublicPermissions(String datasetId)
         if (globusEndpoint.equals("") || globusBasicToken.equals("")) {
             return false;
         }
-        AccessToken clientTokenUser = getClientToken();
+        AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
         if (clientTokenUser == null) {
             logger.severe("Cannot get client token ");
             return false;
@@ -908,7 +915,7 @@ private GlobusTask globusStatusCheck(String taskId, Logger globusLogger) throws
             try {
                 globusLogger.info("checking globus transfer task   " + taskId);
                 Thread.sleep(pollingInterval * 1000);
-                AccessToken clientTokenUser = getClientToken();
+                AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
                 // success = globusServiceBean.getSuccessfulTransfers(clientTokenUser, taskId);
                 task = getTask(clientTokenUser, taskId, globusLogger);
                 if (task != null) {

From f14b75454a524fd8816d6f5367b0e15fbd0ded92 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 9 May 2023 14:53:56 -0400
Subject: [PATCH 050/922] enable globus store main() to run - will revert

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index e5b191f0ed7..4926f59f8a0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -184,7 +184,7 @@
         <dependency>
             <groupId>org.glassfish</groupId>
             <artifactId>jakarta.json</artifactId>
-            <scope>provided</scope>
+            <!--scope>provided</scope-->
             <!-- no version here as managed by Payara BOM above! -->
         </dependency>
         <dependency>

From 51cfa409ddaa7d9d111bff71d6e69beff0a3454c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 9 May 2023 15:53:31 -0400
Subject: [PATCH 051/922] Changes to the edit files page to accommodate "upload
 on create" #9361

---
 .../edu/harvard/iq/dataverse/EditDatafilesPage.java   | 11 ++++++++++-
 .../command/impl/CreateNewDataFilesCommand.java       |  3 ++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index c39e6f62ce2..b892acdb527 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -2058,7 +2058,16 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
             // zip file.
             ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, systemConfig);
             
-            Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null);
+            Command<CreateDataFileResult> cmd;
+            if (mode == FileEditMode.CREATE) {
+                // This is a file upload in the context of creating a brand new
+                // dataset that does not yet exist in the database. We must 
+                // use the version of the Create New Files constructor that takes
+                // the parent Dataverse as the extra argument:
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, null, workingVersion.getDataset().getOwner());
+            } else {
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null);
+            }
             CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
 
         
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index fb4f7a7db86..e2d7f834c4a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -64,7 +64,8 @@
  */
 // Note the commented out @RequiredPermissions. We need to use dynamic 
 // permissions instead, to accommodate both adding files to an existing 
-// dataset and files being uploaded on create of a new dataset. 
+// dataset and files being uploaded in the context of creating a new dataset
+// via the Add Dataset page. 
 //@RequiredPermissions( Permission.EditDataset )
 public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileResult> {
     private static final Logger logger = Logger.getLogger(CreateNewDataFilesCommand.class.getCanonicalName());

From 8560ba4e16f763d48e6ec20f1ddaecd9b5728603 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 9 May 2023 17:16:56 -0400
Subject: [PATCH 052/922] Makes the create file command refuse a zip file if
 uncompressed content exceeds the remaining quota; as opposed to accepting it
 zipped, even if the compressed size under the quota. #9361

---
 .../command/impl/CreateNewDataFilesCommand.java | 17 +++++++++++------
 src/main/java/propertyFiles/Bundle.properties   |  1 +
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index e2d7f834c4a..2608069dcb2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -348,7 +348,11 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                                 if (storageQuotaLimit != null) {
                                     combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize();
                                     if (combinedUnzippedFileSize > storageQuotaLimit) {
-                                        throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                                        //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                                        // change of plans: if the unzipped content inside exceeds the remaining quota, 
+                                        // we reject the upload outright, rather than accepting the zip 
+                                        // file as is. 
+                                        throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.unzipped.quota_exceeded"), bytesToHumanReadable(storageQuotaLimit)), this);
                                     }
                                 }
                             }
@@ -476,11 +480,12 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     logger.warning("One of the unzipped files exceeds the size limit; resorting to saving the file as is. " + femsx.getMessage());
                     warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.size", Arrays.asList(FileSizeChecker.bytesToHumanReadable(fileSizeLimit)));
                     datafiles.clear();
-                } catch (FileExceedsStorageQuotaException fesqx) {
-                    logger.warning("One of the unzipped files exceeds the storage quota limit; resorting to saving the file as is. " + fesqx.getMessage());
-                    warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));
-                    datafiles.clear();
-                } finally {
+                } /*catch (FileExceedsStorageQuotaException fesqx) {
+                    //logger.warning("One of the unzipped files exceeds the storage quota limit; resorting to saving the file as is. " + fesqx.getMessage());
+                    //warningMessage =  BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));
+                    //datafiles.clear();
+                    throw new CommandExecutionException(fesqx.getMessage(), fesqx, this);
+                }*/ finally {
                     if (zipFile != null) {
                         try {
                             zipFile.close();
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 55679a01a07..b12e0df7c08 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2155,6 +2155,7 @@ file.addreplace.file_size_ok=File size is in range.
 file.addreplace.error.byte_abrev=B
 file.addreplace.error.file_exceeds_limit=This file size ({0}) exceeds the size limit of {1}.
 file.addreplace.error.quota_exceeded=This file (size {0}) exceeds the remaining storage quota of {1}.
+file.addreplace.error.unzipped.quota_exceeded=Unzipped files exceed the remaining storage quota of {0}.
 file.addreplace.error.dataset_is_null=The dataset cannot be null.
 file.addreplace.error.dataset_id_is_null=The dataset ID cannot be null.
 file.addreplace.error.parsing=Error in parsing provided json

From 2ac403d35220f6ec61242f7f7f1c12b00f36a93c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 11 May 2023 11:04:00 -0400
Subject: [PATCH 053/922] changes needed to accommodate the quota changing
 dynamically, before the uploaded files are saved (when uploading via the
 page) #9361

---
 .../iq/dataverse/EditDatafilesPage.java       | 25 +++++++-----
 .../datadeposit/MediaResourceManagerImpl.java | 10 +++--
 .../datasetutility/AddReplaceFileHelper.java  |  6 ++-
 .../impl/CreateNewDataFilesCommand.java       | 39 +++++++++++++++----
 4 files changed, 57 insertions(+), 23 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index b892acdb527..57729f321b5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean;
 import edu.harvard.iq.dataverse.DataFile.ChecksumType;
+import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.api.AbstractApiBean;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
@@ -206,6 +207,7 @@ public enum Referrer {
     private final int NUMBER_OF_SCROLL_ROWS = 25;
 
     private DataFile singleFile = null;
+    private UserStorageQuota userStorageQuota = null; 
 
     public DataFile getSingleFile() {
         return singleFile;
@@ -358,7 +360,7 @@ public String getHumanMaxTotalUploadSizeInBytes() {
     }
     
     public boolean isStorageQuotaEnforced() {
-        return maxTotalUploadSizeInBytes != null; 
+        return userStorageQuota != null; 
     }
 
     public Long getMaxIngestSizeInBytes() {
@@ -529,22 +531,24 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo
 
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
         if (systemConfig.isStorageQuotasEnforced()) {
-            this.maxTotalUploadSizeInBytes = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset).getRemainingQuotaInBytes();
+            this.userStorageQuota = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset);
+            this.maxTotalUploadSizeInBytes = userStorageQuota.getRemainingQuotaInBytes();
         } else {
             this.maxTotalUploadSizeInBytes = null; 
         }
         this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
         this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
         this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit();
-
+        
         logger.fine("done");
 
         saveEnabled = true;
+        
         return null;
     }
     
     public boolean isQuotaExceeded() {
-        return systemConfig.isStorageQuotasEnforced() && datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset).getRemainingQuotaInBytes() == 0;
+        return systemConfig.isStorageQuotasEnforced() && userStorageQuota != null && userStorageQuota.getRemainingQuotaInBytes() == 0;
     }
 
     public String init() {
@@ -589,7 +593,8 @@ public String init() {
         clone = workingVersion.cloneDatasetVersion();
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
         if (systemConfig.isStorageQuotasEnforced()) {
-            this.maxTotalUploadSizeInBytes = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset).getRemainingQuotaInBytes();
+            this.userStorageQuota = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset);
+            this.maxTotalUploadSizeInBytes = userStorageQuota.getRemainingQuotaInBytes();
         }
         this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
         this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
@@ -687,7 +692,7 @@ public String init() {
         if (isHasPublicStore()){
             JH.addMessage(FacesMessage.SEVERITY_WARN, getBundleString("dataset.message.label.fileAccess"), getBundleString("dataset.message.publicInstall"));
         }
-
+       
         return null;
     }
 
@@ -1525,7 +1530,7 @@ public void handleDropBoxUpload(ActionEvent event) {
                 // zip file.
                 //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");
                 //CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig);
-                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null);
+                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, dropBoxStream, fileName, "application/octet-stream", null, userStorageQuota, null);
                 CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                 datafiles = createDataFilesResult.getDataFiles();
                 Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
@@ -2064,9 +2069,9 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
                 // dataset that does not yet exist in the database. We must 
                 // use the version of the Create New Files constructor that takes
                 // the parent Dataverse as the extra argument:
-                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null, null, workingVersion.getDataset().getOwner());
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null, null, workingVersion.getDataset().getOwner());
             } else {
-                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, null);
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null);
             }
             CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
 
@@ -2193,7 +2198,7 @@ public void handleExternalUpload() {
                     //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");
                     ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig);
                     
-                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType);
+                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, userStorageQuota, checksumValue, checksumType);
                     CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                     datafiles = createDataFilesResult.getDataFiles();
                     Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
index c71eeb3d375..0f11b858238 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
@@ -10,9 +10,8 @@
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
 import edu.harvard.iq.dataverse.PermissionServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-import edu.harvard.iq.dataverse.dataaccess.StorageIO;
-import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
+import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
@@ -22,7 +21,6 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
-import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -309,7 +307,11 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au
 
             try {
                 //CreateDataFileResult createDataFilesResponse =  FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig);
-                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvReq, editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null);
+                UserStorageQuota quota = null; 
+                if (systemConfig.isStorageQuotasEnforced()) {
+                    quota = dataFileService.getUserStorageQuota(user, dataset);
+                }
+                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvReq, editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, quota, null);
                 CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                 dataFiles = createDataFilesResult.getDataFiles();
             } catch (CommandException ex) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
index 3914c4dc4cc..38a31dba82f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -1212,7 +1212,11 @@ private boolean step_030_createNewFilesViaIngest(){
                     this.newCheckSumType,
                     this.systemConfig);*/
             
-            Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, newCheckSum, newCheckSumType);
+            DataFileServiceBean.UserStorageQuota quota = null; 
+            if (systemConfig.isStorageQuotasEnforced()) {
+                quota = fileService.getUserStorageQuota(dvRequest.getAuthenticatedUser(), dataset);
+            }
+            Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType);
             CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
             initialFileList = createDataFilesResult.getDataFiles();
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 2608069dcb2..b7ceddcd8bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -78,12 +78,13 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private final String newCheckSum; 
     private DataFile.ChecksumType newCheckSumType;
     private final Dataverse dataverse;
+    private final UserStorageQuota quota;
 
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum) {
-        this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, newCheckSum, null);
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum) {
+        this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, null);
     }
     
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, DataFile.ChecksumType newCheckSumType) {
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType) {
         super(aRequest, version.getDataset());
         
         this.version = version;
@@ -94,12 +95,13 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
         this.newCheckSum = newCheckSum; 
         this.newCheckSumType = newCheckSumType;
         this.dataverse = null; 
+        this.quota = quota;
     }
     
     // This version of the command must be used when files are created in the 
     // context of creating a brand new dataset (from the Add Dataset page):
     
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, String newCheckSum, DataFile.ChecksumType newCheckSumType, Dataverse dataverse) {
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Dataverse dataverse) {
         super(aRequest, dataverse);
         
         this.version = version;
@@ -110,11 +112,13 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
         this.newCheckSum = newCheckSum; 
         this.newCheckSumType = newCheckSumType;
         this.dataverse = dataverse;
+        this.quota = quota; 
     }
     
 
     @Override
     public CreateDataFileResult execute(CommandContext ctxt) throws CommandException {
+        logger.info("entering command.execute();");
         List<DataFile> datafiles = new ArrayList<>();
 
         //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default
@@ -132,7 +136,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
         
         if (ctxt.systemConfig().isStorageQuotasEnforced()) {
             //storageQuotaLimit = ctxt.files().getClass()...;
-            UserStorageQuota quota = ctxt.files().getUserStorageQuota(super.getRequest().getAuthenticatedUser(), this.version.getDataset());
+            //UserStorageQuota quota = ctxt.files().getUserStorageQuota(super.getRequest().getAuthenticatedUser(), this.version.getDataset());
             if (quota != null) {
                 storageQuotaLimit = quota.getRemainingQuotaInBytes();
             }
@@ -247,6 +251,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     }
 
                     datafiles.add(datafile);
+                    // Update quota if present
+                    if (quota != null) {
+                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() - datafile.getFilesize());
+                    }
                     return CreateDataFileResult.success(fileName, finalType, datafiles);
                 }
 
@@ -259,6 +267,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 ZipEntry zipEntry = null;
 
                 int fileNumberLimit = ctxt.systemConfig().getZipUploadFilesLimit();
+                Long combinedUnzippedFileSize = 0L;
 
                 try {
                     Charset charset = null;
@@ -307,8 +316,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                      */
 
                     int numberOfUnpackableFiles = 0; 
-                    Long combinedUnzippedFileSize = 0L; 
-
+                     
                     /**
                      * Note that we can't just use zipFile.size(),
                      * unfortunately, since that's the total number of entries,
@@ -363,6 +371,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     
                     // Close the ZipFile, re-open as ZipInputStream: 
                     zipFile.close(); 
+                    // reset:
+                    combinedUnzippedFileSize = 0L;
 
                     if (charset != null) {
                         unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset);
@@ -458,6 +468,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                                         }
 
                                         datafiles.add(datafile);
+                                        combinedUnzippedFileSize += datafile.getFilesize();
                                     }
                                 }
                             }
@@ -505,6 +516,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                         // do nothing - it's just a temp file.
                         logger.warning("Could not remove temp file " + tempFile.getFileName().toString());
                     }
+                    // update the quota object: 
+                    if (quota != null) {
+                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + combinedUnzippedFileSize);
+                    }
                     // and return:
                     return CreateDataFileResult.success(fileName, finalType, datafiles);
                 }
@@ -524,9 +539,9 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     logger.severe("Processing of zipped shapefile failed.");
                     return CreateDataFileResult.error(fileName, finalType);
                 }
+                Long storageQuotaLimitForRezippedFiles = storageQuotaLimit;
 
                 try {
-                    Long storageQuotaLimitForRezippedFiles = storageQuotaLimit;
                     
                     for (File finalFile : shpIngestHelper.getFinalRezippedFiles()) {
                         FileInputStream finalFileInputStream = new FileInputStream(finalFile);
@@ -598,6 +613,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                         logger.warning("Unable to delete: " + tempFile.toString() + "due to Security Exception: "
                                 + se.getMessage());
                     }
+                    // update the quota object: 
+                    if (quota != null) {
+                        quota.setTotalUsageInBytes(storageQuotaLimitForRezippedFiles);
+                    }
                     return CreateDataFileResult.success(fileName, finalType, datafiles);
                 } else {
                     logger.severe("No files added from directory of rezipped shapefiles");
@@ -665,6 +684,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             }
             datafiles.add(datafile);
 
+            // Update quota (may not be necessary in the context of direct upload - ?)
+            if (quota != null) {
+                quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() - datafile.getFilesize());
+            }
             return CreateDataFileResult.success(fileName, finalType, datafiles);
         }
 

From 87c7383c646de0756503cc6116e13f3d47f0ad31 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 11 May 2023 13:03:07 -0400
Subject: [PATCH 054/922] More fixes for the command. #9361

---
 .../impl/CreateNewDataFilesCommand.java       | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index b7ceddcd8bf..6f582a4c015 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -118,7 +118,6 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
 
     @Override
     public CreateDataFileResult execute(CommandContext ctxt) throws CommandException {
-        logger.info("entering command.execute();");
         List<DataFile> datafiles = new ArrayList<>();
 
         //When there is no checksum/checksumtype being sent (normal upload, needs to be calculated), set the type to the current default
@@ -135,8 +134,6 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
         Long storageQuotaLimit = null; 
         
         if (ctxt.systemConfig().isStorageQuotasEnforced()) {
-            //storageQuotaLimit = ctxt.files().getClass()...;
-            //UserStorageQuota quota = ctxt.files().getUserStorageQuota(super.getRequest().getAuthenticatedUser(), this.version.getDataset());
             if (quota != null) {
                 storageQuotaLimit = quota.getRemainingQuotaInBytes();
             }
@@ -220,9 +217,11 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 }
 
                 DataFile datafile = null;
+                long fileSize = 0L; 
                 try {
                     uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));
                     File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit, storageQuotaLimit);
+                    fileSize = unZippedTempFile.length();
                     datafile = FileUtil.createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, ctxt.systemConfig().getFileFixityChecksumAlgorithm());
                 } catch (IOException | FileExceedsMaxSizeException | FileExceedsStorageQuotaException ioex) {
                     // it looks like we simply skip the file silently, if its uncompressed size
@@ -253,7 +252,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     datafiles.add(datafile);
                     // Update quota if present
                     if (quota != null) {
-                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() - datafile.getFilesize());
+                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
                     }
                     return CreateDataFileResult.success(fileName, finalType, datafiles);
                 }
@@ -539,7 +538,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     logger.severe("Processing of zipped shapefile failed.");
                     return CreateDataFileResult.error(fileName, finalType);
                 }
-                Long storageQuotaLimitForRezippedFiles = storageQuotaLimit;
+                long combinedRezippedFileSize = 0L;
 
                 try {
                     
@@ -551,7 +550,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                             continue;
                         }
 
-                        File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit, storageQuotaLimitForRezippedFiles);
+                        File unZippedShapeTempFile = saveInputStreamInTempFile(finalFileInputStream, fileSizeLimit, storageQuotaLimit != null ? storageQuotaLimit - combinedRezippedFileSize : null);
                         DataFile new_datafile = FileUtil.createSingleDataFile(version, unZippedShapeTempFile, finalFile.getName(), finalType, ctxt.systemConfig().getFileFixityChecksumAlgorithm());
                         
                         String directoryName = null;
@@ -569,10 +568,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                         }
                         if (new_datafile != null) {
                             datafiles.add(new_datafile);
+                            combinedRezippedFileSize += unZippedShapeTempFile.length();
                             // todo: can this new_datafile be null?
-                            if (storageQuotaLimitForRezippedFiles != null) {
-                                storageQuotaLimitForRezippedFiles = storageQuotaLimitForRezippedFiles - new_datafile.getFilesize();
-                            }
                         } else {
                             logger.severe("Could not add part of rezipped shapefile. new_datafile was null: " + finalFile.getName());
                         }
@@ -615,7 +612,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     }
                     // update the quota object: 
                     if (quota != null) {
-                        quota.setTotalUsageInBytes(storageQuotaLimitForRezippedFiles);
+                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + combinedRezippedFileSize);
                     }
                     return CreateDataFileResult.success(fileName, finalType, datafiles);
                 } else {
@@ -686,7 +683,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
 
             // Update quota (may not be necessary in the context of direct upload - ?)
             if (quota != null) {
-                quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() - datafile.getFilesize());
+                quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
             }
             return CreateDataFileResult.success(fileName, finalType, datafiles);
         }

From dbd97ff1c9c7c19ec56270fdfcef7e11056a7778 Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Thu, 11 May 2023 16:38:49 -0400
Subject: [PATCH 055/922] develop + title

---
 .../api/imports/ImportDDIServiceBean.java     |  33 +-
 .../dataverse/export/ddi/DdiExportUtil.java   | 481 +++++++++++-------
 .../export/openaire/OpenAireExportUtil.java   | 243 +++------
 3 files changed, 393 insertions(+), 364 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index 458803e0c92..bafd7267acb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -1266,24 +1266,26 @@ private void parseVersionNumber(DatasetVersionDTO dvDTO, String versionNumber) {
 
     }
    
-   private void processSerStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException {
-          FieldDTO seriesName=null;
-          FieldDTO seriesInformation=null;
-          for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+    private void processSerStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException {
+        FieldDTO seriesInformation = null;
+        FieldDTO seriesName = null;
+        for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {            
             if (event == XMLStreamConstants.START_ELEMENT) {
+                if (xmlr.getLocalName().equals("serInfo")) {
+                     seriesInformation = FieldDTO.createPrimitiveFieldDTO("seriesInformation", parseText(xmlr));
+                }
                 if (xmlr.getLocalName().equals("serName")) {
-                   seriesName = FieldDTO.createPrimitiveFieldDTO("seriesName", parseText(xmlr));
-                  
-                } else if (xmlr.getLocalName().equals("serInfo")) {
-                    seriesInformation=FieldDTO.createPrimitiveFieldDTO("seriesInformation", parseText(xmlr) );
+                     seriesName = FieldDTO.createPrimitiveFieldDTO("seriesName", parseText(xmlr));
                 }
             } else if (event == XMLStreamConstants.END_ELEMENT) {
                 if (xmlr.getLocalName().equals("serStmt")) {
-                    citation.getFields().add(FieldDTO.createCompoundFieldDTO("series",seriesName,seriesInformation ));
+                    if (seriesInformation != null || seriesName != null) {
+                        citation.addField(FieldDTO.createMultipleCompoundFieldDTO("series", seriesName, seriesInformation ));
+                    }
                     return;
                 }
             }
-        }
+        }     
     }
 
     private void processDistStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException {
@@ -1337,7 +1339,6 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
         List<HashSet<FieldDTO>> producers = new ArrayList<>();
         List<HashSet<FieldDTO>> grants = new ArrayList<>();
         List<HashSet<FieldDTO>> software = new ArrayList<>();
-        List<String> prodPlac = new ArrayList<>();
 
         for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
             if (event == XMLStreamConstants.START_ELEMENT) {
@@ -1353,7 +1354,9 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
                 } else if (xmlr.getLocalName().equals("prodDate")) {
                     citation.getFields().add(FieldDTO.createPrimitiveFieldDTO("productionDate", parseDate(xmlr, "prodDate")));
                 } else if (xmlr.getLocalName().equals("prodPlac")) {
+                    List<String> prodPlac = new ArrayList<>();
                     prodPlac.add(parseText(xmlr, "prodPlac"));
+                    citation.getFields().add(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.productionPlace, prodPlac));
                 } else if (xmlr.getLocalName().equals("software")) {
                     HashSet<FieldDTO> set = new HashSet<>();
                     addToSet(set,"softwareVersion", xmlr.getAttributeValue(null, "version"));
@@ -1386,9 +1389,6 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
                     if (producers.size()>0) {
                         citation.getFields().add(FieldDTO.createMultipleCompoundFieldDTO("producer", producers));
                     }
-                    if (prodPlac.size() > 0) {
-                        citation.getFields().add(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.productionPlace, prodPlac));
-                    }
                     return;
                 }
             }
@@ -1437,8 +1437,9 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws
                     if (otherIds.size()>0) {
                         citation.addField(FieldDTO.createMultipleCompoundFieldDTO("otherId", otherIds));
                     }
-                    if (altTitles.size()>0) {
-                        citation.addField(FieldDTO.createMultiplePrimitiveFieldDTO("alternativeTitle", altTitles));
+                    if (!altTitles.isEmpty()) {
+                        FieldDTO field = FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.alternativeTitle, altTitles);
+                        citation.getFields().add(field);
                     }
                     return;
                 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index 4ea90ea6199..a647c2a6f2a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -30,6 +30,7 @@
 import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_TAG;
 import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_UNF;
 import edu.harvard.iq.dataverse.export.DDIExporter;
+import edu.harvard.iq.dataverse.pidproviders.PidUtil;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
 
@@ -93,7 +94,6 @@ public class DdiExportUtil {
     public static final String CITATION_BLOCK_NAME = "citation";
 
     public static String datasetDtoAsJson2ddi(String datasetDtoAsJson) {
-        logger.fine(JsonUtil.prettyPrint(datasetDtoAsJson));
         Gson gson = new Gson();
         DatasetDTO datasetDto = gson.fromJson(datasetDtoAsJson, DatasetDTO.class);
         try {
@@ -181,7 +181,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
         String pidUri = pid;
         //Some tests don't send real PIDs - don't try to get their URL form
         if(!pidUri.equals("null:null/null")) {
-            pidUri= new GlobalId(persistentProtocol + ":" + persistentAuthority + "/" + persistentId).toURL().toString();
+            pidUri= PidUtil.parseAsGlobalID(persistentProtocol, persistentAuthority, persistentId).asURL();
         }
         // The "persistentAgency" tag is used for the "agency" attribute of the 
         // <IDNo> ddi section; back in the DVN3 days we used "handle" and "DOI" 
@@ -202,12 +202,12 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
        
         writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage());
         writeFullElement(xmlw, "subTitl", dto2Primitive(version, DatasetFieldConstant.subTitle));
-       
+        //writeFullElement(xmlw, "altTitl", dto2Primitive(version, DatasetFieldConstant.alternativeTitle));
         FieldDTO altField = dto2FieldDTO( version, DatasetFieldConstant.alternativeTitle, "citation"  );
         if (altField != null) {
             writeMultipleElement(xmlw, "altTitl", altField, datasetDto.getMetadataLanguage());
         }
-
+        
         xmlw.writeStartElement("IDNo");
         writeAttribute(xmlw, "agency", persistentAgency);
         
@@ -239,9 +239,11 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
         }
         writeDistributorsElement(xmlw, version, datasetDto.getMetadataLanguage());
         writeContactsElement(xmlw, version);
-        writeFullElement(xmlw, "distDate", dto2Primitive(version, DatasetFieldConstant.distributionDate));
+        /* per SCHEMA, depositr comes before depDate! - L.A. */
         writeFullElement(xmlw, "depositr", dto2Primitive(version, DatasetFieldConstant.depositor));
+        /* ... and depDate comes before distDate - L.A. */
         writeFullElement(xmlw, "depDate", dto2Primitive(version, DatasetFieldConstant.dateOfDeposit));
+        writeFullElement(xmlw, "distDate", dto2Primitive(version, DatasetFieldConstant.distributionDate));
 
         xmlw.writeEndElement(); // diststmt
 
@@ -294,23 +296,16 @@ private static void writeOtherStudyMaterial(XMLStreamWriter xmlw , DatasetVersio
         xmlw.writeEndElement(); //othrStdyMat
     }
 
+    /*
+            <xs:sequence>
+               <xs:element ref="setAvail" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="useStmt" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="notes" minOccurs="0" maxOccurs="unbounded"/>
+            </xs:sequence>
+    */
     private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException {
         xmlw.writeStartElement("dataAccs");
-        if (version.getTermsOfUse() != null && !version.getTermsOfUse().trim().equals("")) {
-            xmlw.writeStartElement("notes");
-            writeAttribute(xmlw, "type", NOTE_TYPE_TERMS_OF_USE);
-            writeAttribute(xmlw, "level", LEVEL_DV);
-            xmlw.writeCharacters(version.getTermsOfUse());
-            xmlw.writeEndElement(); //notes
-        }
-        if (version.getTermsOfAccess() != null && !version.getTermsOfAccess().trim().equals("")) {
-            xmlw.writeStartElement("notes");
-            writeAttribute(xmlw, "type", NOTE_TYPE_TERMS_OF_ACCESS);
-            writeAttribute(xmlw, "level", LEVEL_DV);
-            xmlw.writeCharacters(version.getTermsOfAccess());
-            xmlw.writeEndElement(); //notes
-        }
-
+        
         xmlw.writeStartElement("setAvail");
         writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace());
         writeFullElement(xmlw, "origArch", version.getOriginalArchive());
@@ -318,6 +313,7 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver
         writeFullElement(xmlw, "collSize", version.getSizeOfCollection());
         writeFullElement(xmlw, "complete", version.getStudyCompletion());
         xmlw.writeEndElement(); //setAvail
+        
         xmlw.writeStartElement("useStmt");
         writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration());
         writeFullElement(xmlw, "specPerm", version.getSpecialPermissions());
@@ -328,6 +324,15 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver
         writeFullElement(xmlw, "conditions", version.getConditions());
         writeFullElement(xmlw, "disclaimer", version.getDisclaimer());
         xmlw.writeEndElement(); //useStmt
+        
+        /* any <note>s: */
+        if (version.getTermsOfAccess() != null && !version.getTermsOfAccess().trim().equals("")) {
+            xmlw.writeStartElement("notes");
+            writeAttribute(xmlw, "type", NOTE_TYPE_TERMS_OF_ACCESS);
+            writeAttribute(xmlw, "level", LEVEL_DV);
+            xmlw.writeCharacters(version.getTermsOfAccess());
+            xmlw.writeEndElement(); //notes
+        }
         xmlw.writeEndElement(); //dataAccs
     }
     
@@ -388,141 +393,222 @@ private static void writeVersionStatement(XMLStreamWriter xmlw, DatasetVersionDT
         xmlw.writeEndElement(); // verStmt
     }
     
+    /* From the DDI 2.5 schema: 
+            <xs:sequence>
+               <xs:element ref="timePrd" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="collDate" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="nation" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="geogCover" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="geogUnit" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="geoBndBox" minOccurs="0"/>
+               <xs:element ref="boundPoly" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="anlyUnit" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="universe" minOccurs="0" maxOccurs="unbounded"/>
+               <xs:element ref="dataKind" minOccurs="0" maxOccurs="unbounded"/>
+            </xs:sequence>
+    */
     private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, DatasetVersionDTO datasetVersionDTO, String lang) throws XMLStreamException {
         xmlw.writeStartElement("sumDscr");
+        FieldDTO timePeriodCoveredDTO = null;
+        FieldDTO dateOfCollectionDTO = null;
+        FieldDTO geographicCoverageDTO = null;
+        FieldDTO geographicBoundingBoxDTO = null;
+        FieldDTO unitOfAnalysisDTO = null;
+        FieldDTO universeDTO = null;
+        FieldDTO kindOfDataDTO = null;
+
         for (Map.Entry<String, MetadataBlockDTO> entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
             String key = entry.getKey();
             MetadataBlockDTO value = entry.getValue();
+
             if ("citation".equals(key)) {
-                Integer per = 0;
-                Integer coll = 0;
                 for (FieldDTO fieldDTO : value.getFields()) {
                     if (DatasetFieldConstant.timePeriodCovered.equals(fieldDTO.getTypeName())) {
-                        String dateValStart = "";
-                        String dateValEnd = "";
-                        for (HashSet<FieldDTO> foo : fieldDTO.getMultipleCompound()) {
-                            per++;
-                            for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
-                                FieldDTO next = iterator.next();
-                                if (DatasetFieldConstant.timePeriodCoveredStart.equals(next.getTypeName())) {
-                                    dateValStart = next.getSinglePrimitive();
-                                }
-                                if (DatasetFieldConstant.timePeriodCoveredEnd.equals(next.getTypeName())) {
-                                    dateValEnd = next.getSinglePrimitive();
-                                }
-                            }
-                            if (!dateValStart.isEmpty()) {
-                                writeDateElement(xmlw, "timePrd", "P"+ per.toString(), "start", dateValStart );
-                            }
-                            if (!dateValEnd.isEmpty()) {
-                                writeDateElement(xmlw, "timePrd",  "P"+ per.toString(), "end", dateValEnd );
-                            }
-                        }
+                        timePeriodCoveredDTO = fieldDTO;
                     }
+
                     if (DatasetFieldConstant.dateOfCollection.equals(fieldDTO.getTypeName())) {
-                        String dateValStart = "";
-                        String dateValEnd = "";
-                        for (HashSet<FieldDTO> foo : fieldDTO.getMultipleCompound()) {
-                            coll++;
-                            for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
-                                FieldDTO next = iterator.next();
-                                if (DatasetFieldConstant.dateOfCollectionStart.equals(next.getTypeName())) {
-                                    dateValStart = next.getSinglePrimitive();
-                                }
-                                if (DatasetFieldConstant.dateOfCollectionEnd.equals(next.getTypeName())) {
-                                    dateValEnd = next.getSinglePrimitive();
-                                }
-                            }
-                            if (!dateValStart.isEmpty()) {
-                                writeDateElement(xmlw, "collDate",  "P"+ coll.toString(), "start", dateValStart );
-                            }
-                            if (!dateValEnd.isEmpty()) {
-                                writeDateElement(xmlw,  "collDate",  "P"+ coll.toString(), "end", dateValEnd );
-                            }
-                        }
+                        dateOfCollectionDTO = fieldDTO;
                     }
+
                     if (DatasetFieldConstant.kindOfData.equals(fieldDTO.getTypeName())) {
-                        writeMultipleElement(xmlw, "dataKind", fieldDTO, lang);
+                        kindOfDataDTO = fieldDTO;
                     }
                 }
             }
-            
-            if("geospatial".equals(key)){                
+
+            if ("geospatial".equals(key)) {
                 for (FieldDTO fieldDTO : value.getFields()) {
                     if (DatasetFieldConstant.geographicCoverage.equals(fieldDTO.getTypeName())) {
-
-                        for (HashSet<FieldDTO> foo : fieldDTO.getMultipleCompound()) {
-                            HashMap<String, String> geoMap = new HashMap<>();
-                            for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
-                                FieldDTO next = iterator.next();
-                                if (DatasetFieldConstant.country.equals(next.getTypeName())) {
-                                    geoMap.put("country", next.getSinglePrimitive());
-                                }
-                                if (DatasetFieldConstant.city.equals(next.getTypeName())) {
-                                    geoMap.put("city", next.getSinglePrimitive());
-                                }
-                                if (DatasetFieldConstant.state.equals(next.getTypeName())) {
-                                    geoMap.put("state", next.getSinglePrimitive());
-                                } 
-                                if (DatasetFieldConstant.otherGeographicCoverage.equals(next.getTypeName())) {
-                                    geoMap.put("otherGeographicCoverage", next.getSinglePrimitive());
-                                } 
-                            }
-
-                            if (geoMap.get("country") != null) {
-                                writeFullElement(xmlw, "nation", geoMap.get("country"));
-                            }
-                            if (geoMap.get("city") != null) {
-                                writeFullElement(xmlw, "geogCover", geoMap.get("city"));
-                            }
-                            if (geoMap.get("state") != null) {
-                                writeFullElement(xmlw, "geogCover", geoMap.get("state"));
-                            }
-                            if (geoMap.get("otherGeographicCoverage") != null) {
-                                writeFullElement(xmlw, "geogCover", geoMap.get("otherGeographicCoverage"));
-                            }
-
-                        }
+                        geographicCoverageDTO = fieldDTO;
                     }
                     if (DatasetFieldConstant.geographicBoundingBox.equals(fieldDTO.getTypeName())) {
 
-                        for (HashSet<FieldDTO> foo : fieldDTO.getMultipleCompound()) {
-                            xmlw.writeStartElement("geoBndBox");
-                            for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
-                                FieldDTO next = iterator.next();
-                                if (DatasetFieldConstant.westLongitude.equals(next.getTypeName())) {
-                                    writeFullElement(xmlw, "westBL", next.getSinglePrimitive());
-                                }
-                                if (DatasetFieldConstant.eastLongitude.equals(next.getTypeName())) {
-                                    writeFullElement(xmlw, "eastBL", next.getSinglePrimitive());
-                                }
-                                if (DatasetFieldConstant.northLatitude.equals(next.getTypeName())) {
-                                    writeFullElement(xmlw, "northBL", next.getSinglePrimitive());
-                                }  
-                                if (DatasetFieldConstant.southLatitude.equals(next.getTypeName())) {
-                                    writeFullElement(xmlw, "southBL", next.getSinglePrimitive());
-                                }
-
-                            }
-                            xmlw.writeEndElement();
-                        }
+                        geographicBoundingBoxDTO = fieldDTO;
 
                     }
                 }
-                    writeFullElementList(xmlw, "geogUnit", dto2PrimitiveList(datasetVersionDTO, DatasetFieldConstant.geographicUnit));
             }
 
-            if("socialscience".equals(key)){                
+            if ("socialscience".equals(key)) {
                 for (FieldDTO fieldDTO : value.getFields()) {
                     if (DatasetFieldConstant.universe.equals(fieldDTO.getTypeName())) {
-                        writeMultipleElement(xmlw, "universe", fieldDTO, lang);
+                        universeDTO = fieldDTO;
                     }
                     if (DatasetFieldConstant.unitOfAnalysis.equals(fieldDTO.getTypeName())) {
-                        writeI18NElementList(xmlw, "anlyUnit", fieldDTO.getMultipleVocab(), "unitOfAnalysis", fieldDTO.getTypeClass(), "socialscience", lang);
+                        unitOfAnalysisDTO = fieldDTO;
+                    }
+                }
+            }
+        }
+        /* Finally, we can write the fields we have collected, in the correct order: -L.A.*/
+
+        if (timePeriodCoveredDTO != null) {
+            String dateValStart = "";
+            String dateValEnd = "";
+            Integer per = 0;
+            for (HashSet<FieldDTO> foo : timePeriodCoveredDTO.getMultipleCompound()) {
+                per++;
+                for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
+                    FieldDTO next = iterator.next();
+                    if (DatasetFieldConstant.timePeriodCoveredStart.equals(next.getTypeName())) {
+                        dateValStart = next.getSinglePrimitive();
+                    }
+                    if (DatasetFieldConstant.timePeriodCoveredEnd.equals(next.getTypeName())) {
+                        dateValEnd = next.getSinglePrimitive();
+                    }
+                }
+                if (!dateValStart.isEmpty()) {
+                    writeDateElement(xmlw, "timePrd", "P" + per.toString(), "start", dateValStart);
+                }
+                if (!dateValEnd.isEmpty()) {
+                    writeDateElement(xmlw, "timePrd", "P" + per.toString(), "end", dateValEnd);
+                }
+            }
+        }
+
+        if (dateOfCollectionDTO != null) {
+            String dateValStart = "";
+            String dateValEnd = "";
+            Integer coll = 0;
+            for (HashSet<FieldDTO> foo : dateOfCollectionDTO.getMultipleCompound()) {
+                coll++;
+                for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
+                    FieldDTO next = iterator.next();
+                    if (DatasetFieldConstant.dateOfCollectionStart.equals(next.getTypeName())) {
+                        dateValStart = next.getSinglePrimitive();
+                    }
+                    if (DatasetFieldConstant.dateOfCollectionEnd.equals(next.getTypeName())) {
+                        dateValEnd = next.getSinglePrimitive();
                     }
                 }
+                if (!dateValStart.isEmpty()) {
+                    writeDateElement(xmlw, "collDate", "P" + coll.toString(), "start", dateValStart);
+                }
+                if (!dateValEnd.isEmpty()) {
+                    writeDateElement(xmlw, "collDate", "P" + coll.toString(), "end", dateValEnd);
+                }
             }
         }
+
+        /* <nation> and <geogCover> come next, in that order. -L.A. */
+        if (geographicCoverageDTO != null) {
+
+            List<String> nationList = new ArrayList<>();
+            List<String> geogCoverList = new ArrayList<>();
+
+            for (HashSet<FieldDTO> foo : geographicCoverageDTO.getMultipleCompound()) {
+                for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
+                    FieldDTO next = iterator.next();
+                    /* our "country" field maps 1:1 to the DDI "<nation>": */
+                    if (DatasetFieldConstant.country.equals(next.getTypeName())) {
+                        nationList.add(next.getSinglePrimitive());
+                    }
+                    /* city, state and otherGeographicCoverage all exported as "<geogCover>": */
+                    if (DatasetFieldConstant.city.equals(next.getTypeName())
+                            || DatasetFieldConstant.state.equals(next.getTypeName())
+                            || DatasetFieldConstant.otherGeographicCoverage.equals(next.getTypeName())) {
+                        geogCoverList.add(next.getSinglePrimitive());
+                    }
+                }
+            }
+
+            /**
+             * And now we can write all the fields encountered, first the
+             * "<nation>" entries, then all the "<geogCover>" ones:
+             */
+            for (String nationEntry : nationList) {
+                writeFullElement(xmlw, "nation", nationEntry);
+            }
+            for (String geogCoverEntry : geogCoverList) {
+                writeFullElement(xmlw, "geogCover", geogCoverEntry);
+            }
+        }
+
+        writeFullElementList(xmlw, "geogUnit", dto2PrimitiveList(datasetVersionDTO, DatasetFieldConstant.geographicUnit));
+
+        /* Only 1 geoBndBox is allowed in the DDI.
+           So, I'm just going to arbitrarily use the first one, and ignore the rest! -L.A. */
+        if (geographicBoundingBoxDTO != null) {
+            HashSet<FieldDTO> bndBoxSet = geographicBoundingBoxDTO.getMultipleCompound().get(0);
+            xmlw.writeStartElement("geoBndBox");
+            HashMap<String, String> geoBndBoxMap = new HashMap<>();
+            for (FieldDTO next : bndBoxSet) {
+                if (DatasetFieldConstant.westLongitude.equals(next.getTypeName())) {
+                    geoBndBoxMap.put("westBL", next.getSinglePrimitive());
+                }
+                if (DatasetFieldConstant.eastLongitude.equals(next.getTypeName())) {
+                    geoBndBoxMap.put("eastBL", next.getSinglePrimitive());
+                }
+                if (DatasetFieldConstant.northLatitude.equals(next.getTypeName())) {
+                    geoBndBoxMap.put("northBL", next.getSinglePrimitive());
+                }
+                if (DatasetFieldConstant.southLatitude.equals(next.getTypeName())) {
+                    geoBndBoxMap.put("southBL", next.getSinglePrimitive());
+                }
+            }
+
+            /* Once again, order is important! */
+ /*
+                        <xs:sequence>
+                            <xs:element ref="westBL"/>
+                            <xs:element ref="eastBL"/>
+                            <xs:element ref="southBL"/>
+                            <xs:element ref="northBL"/>
+                        </xs:sequence>
+             */
+            if (geoBndBoxMap.get("westBL") != null) {
+                writeFullElement(xmlw, "westBL", geoBndBoxMap.get("westBL"));
+            }
+            if (geoBndBoxMap.get("eastBL") != null) {
+                writeFullElement(xmlw, "eastBL", geoBndBoxMap.get("eastBL"));
+            }
+            if (geoBndBoxMap.get("southBL") != null) {
+                writeFullElement(xmlw, "southBL", geoBndBoxMap.get("southBL"));
+            }
+            if (geoBndBoxMap.get("northBL") != null) {
+                writeFullElement(xmlw, "northBL", geoBndBoxMap.get("northBL"));
+            }
+
+            xmlw.writeEndElement();
+        }
+
+        /* analyUnit: */
+        if (unitOfAnalysisDTO != null) {
+            writeI18NElementList(xmlw, "anlyUnit", unitOfAnalysisDTO.getMultipleVocab(), "unitOfAnalysis", unitOfAnalysisDTO.getTypeClass(), "socialscience", lang);
+
+        }
+
+        /* universe: */
+        if (universeDTO != null) {
+            writeMultipleElement(xmlw, "universe", universeDTO, lang);
+        }
+
+        /* finally, any "kind of data" entries: */
+        if (kindOfDataDTO != null) {
+            writeMultipleElement(xmlw, "dataKind", kindOfDataDTO, lang);
+        }
+
         xmlw.writeEndElement(); //sumDscr     
     }
     
@@ -544,6 +630,29 @@ private static void writeDateElement(XMLStreamWriter xmlw, String element, Strin
 
     }
     
+    /**
+     * Again, <dataColl> is an xs:sequence - order is important and must follow
+     * the schema. -L.A.
+     * <xs:sequence>
+     * <xs:element ref="timeMeth" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="dataCollector" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="collectorTraining" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="frequenc" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="sampProc" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="sampleFrame" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="targetSampleSize" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="deviat" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="collMode" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="resInstru" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="instrumentDevelopment" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="sources" minOccurs="0"/>
+     * <xs:element ref="collSitu" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="actMin" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="ConOps" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="weight" minOccurs="0" maxOccurs="unbounded"/>
+     * <xs:element ref="cleanOps" minOccurs="0" maxOccurs="unbounded"/>
+     * </xs:sequence>
+     */
     private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO version, String lang) throws XMLStreamException{
         xmlw.writeStartElement("method");
         xmlw.writeStartElement("dataColl");
@@ -557,13 +666,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO
 
         writeI18NElement(xmlw, "deviat", version, DatasetFieldConstant.deviationsFromSampleDesign, lang);
 
-        xmlw.writeStartElement("sources");
-        writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources));
-        writeI18NElement(xmlw, "srcOrig", version, DatasetFieldConstant.originOfSources, lang);
-        writeI18NElement(xmlw, "srcChar", version, DatasetFieldConstant.characteristicOfSources, lang);
-        writeI18NElement(xmlw, "srcDocu", version, DatasetFieldConstant.accessToSources, lang);
-        xmlw.writeEndElement(); //sources
-
+        /* <collMode> comes before <sources>: */
         FieldDTO collModeFieldDTO = dto2FieldDTO(version, DatasetFieldConstant.collectionMode, "socialscience");
         if (collModeFieldDTO != null) {
             // This field was made multiple as of 5.10
@@ -575,21 +678,33 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO
                 writeI18NElement(xmlw, "collMode", version, DatasetFieldConstant.collectionMode, lang);
             }
         }
+        /* and so does <resInstru>: */
         writeI18NElement(xmlw, "resInstru", version, DatasetFieldConstant.researchInstrument, lang); 
+        xmlw.writeStartElement("sources");
+        writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources));
+        writeI18NElement(xmlw, "srcOrig", version, DatasetFieldConstant.originOfSources, lang);
+        writeI18NElement(xmlw, "srcChar", version, DatasetFieldConstant.characteristicOfSources, lang);
+        writeI18NElement(xmlw, "srcDocu", version, DatasetFieldConstant.accessToSources, lang);
+        xmlw.writeEndElement(); //sources
+
+        
         writeI18NElement(xmlw, "collSitu", version, DatasetFieldConstant.dataCollectionSituation, lang);
         writeI18NElement(xmlw, "actMin", version, DatasetFieldConstant.actionsToMinimizeLoss, lang);
-        writeI18NElement(xmlw, "conOps", version, DatasetFieldConstant.controlOperations, lang);
+        /* "<ConOps>" has the uppercase C: */
+        writeI18NElement(xmlw, "ConOps", version, DatasetFieldConstant.controlOperations, lang);
         writeI18NElement(xmlw, "weight", version, DatasetFieldConstant.weighting, lang);  
         writeI18NElement(xmlw, "cleanOps", version, DatasetFieldConstant.cleaningOperations, lang);
 
         xmlw.writeEndElement(); //dataColl
+        /* <notes> before <anlyInfo>: */
+        writeNotesElement(xmlw, version);
+
         xmlw.writeStartElement("anlyInfo");
         //writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes));
         writeI18NElement(xmlw, "respRate", version, DatasetFieldConstant.responseRate, lang);
         writeI18NElement(xmlw, "EstSmpErr", version, DatasetFieldConstant.samplingErrorEstimates, lang);
         writeI18NElement(xmlw, "dataAppr", version, DatasetFieldConstant.otherDataAppraisal, lang); 
         xmlw.writeEndElement(); //anlyInfo
-        writeNotesElement(xmlw, version);
         
         xmlw.writeEndElement();//method
     }
@@ -852,7 +967,6 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT
                             String producerAffiliation = "";
                             String producerAbbreviation = "";
                             String producerLogo = "";
-                            String producerURL = "";
                             for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
                                 FieldDTO next = iterator.next();
                                 if (DatasetFieldConstant.producerName.equals(next.getTypeName())) {
@@ -867,10 +981,6 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT
                                 if (DatasetFieldConstant.producerLogo.equals(next.getTypeName())) {
                                     producerLogo = next.getSinglePrimitive();
                                 }
-                                if (DatasetFieldConstant.producerURL.equals(next.getTypeName())) {
-                                    producerURL = next.getSinglePrimitive();
-
-                                }
                             }
                             if (!producerName.isEmpty()) {
                                 xmlw.writeStartElement("producer");
@@ -880,12 +990,9 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT
                                 if (!producerAbbreviation.isEmpty()) {
                                     writeAttribute(xmlw, "abbr", producerAbbreviation);
                                 }
-                                if (!producerLogo.isEmpty()) {
+                                /*if (!producerLogo.isEmpty()) {
                                     writeAttribute(xmlw, "role", producerLogo);
-                                }
-                                if (!producerURL.isEmpty()) {
-                                    writeAttribute(xmlw, "URI", producerURL);
-                                }
+                                }*/
                                 xmlw.writeCharacters(producerName);
                                 xmlw.writeEndElement(); //AuthEnty
                             }
@@ -896,12 +1003,10 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT
             }
         }
         writeFullElement(xmlw, "prodDate", dto2Primitive(version, DatasetFieldConstant.productionDate));
-
-        FieldDTO  prodPlac = dto2FieldDTO( version, DatasetFieldConstant.productionPlace, "citation"  );
-        if (prodPlac != null) {
-            writeMultipleElement(xmlw, "prodPlac", prodPlac, null);
-        }
-
+        // productionPlace was made multiple as of 5.14:
+        // (a quick backward compatibility check was added to dto2PrimitiveList(),
+        // see the method for details)
+        writeFullElementList(xmlw, "prodPlac", dto2PrimitiveList(version, DatasetFieldConstant.productionPlace));        
         writeSoftwareElement(xmlw, version);
   
         writeGrantElement(xmlw, version);
@@ -921,7 +1026,6 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio
                             String distributorAffiliation = "";
                             String distributorAbbreviation = "";
                             String distributorURL = "";
-                            String distributorLogoURL = "";
                             for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
                                 FieldDTO next = iterator.next();
                                 if (DatasetFieldConstant.distributorName.equals(next.getTypeName())) {
@@ -936,9 +1040,6 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio
                                 if (DatasetFieldConstant.distributorURL.equals(next.getTypeName())) {
                                     distributorURL = next.getSinglePrimitive();
                                 }
-                                if (DatasetFieldConstant.distributorLogo.equals(next.getTypeName())) {
-                                    distributorLogoURL = next.getSinglePrimitive();
-                                }
                             }
                             if (!distributorName.isEmpty()) {
                                 xmlw.writeStartElement("distrbtr");
@@ -954,9 +1055,6 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio
                                 if (!distributorURL.isEmpty()) {
                                     writeAttribute(xmlw, "URI", distributorURL);
                                 }
-                                if (!distributorLogoURL.isEmpty()) {
-                                    writeAttribute(xmlw, "role", distributorLogoURL);
-                                }
                                 xmlw.writeCharacters(distributorName);
                                 xmlw.writeEndElement(); //AuthEnty
                             }
@@ -1000,16 +1098,33 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO
                             if (citation != null && !citation.trim().equals("")) {
                                 xmlw.writeStartElement("relPubl");
                                 xmlw.writeStartElement("citation");
+                                /* <xs:sequence>
+                                    <xs:element ref="titlStmt"/>
+                                    <xs:element ref="rspStmt" minOccurs="0"/>
+                                    <xs:element ref="prodStmt" minOccurs="0"/>
+                                    <xs:element ref="distStmt" minOccurs="0"/>
+                                    <xs:element ref="serStmt" minOccurs="0" maxOccurs="unbounded"/>
+                                    <xs:element ref="verStmt" minOccurs="0" maxOccurs="unbounded"/>
+                                    <xs:element ref="biblCit" minOccurs="0" maxOccurs="unbounded"/>
+                                    <xs:element ref="holdings" minOccurs="0" maxOccurs="unbounded"/>
+                                    <xs:element ref="notes" minOccurs="0" maxOccurs="unbounded"/>
+                                    <xs:group ref="dc:elementsAndRefinementsGroup"/>
+                                   </xs:sequence>
+                                 (In other words - titlStmt is mandatory! -L.A.)
+                                */
+                                xmlw.writeStartElement("titlStmt");
+                                writeFullElement(xmlw, "titl", citation);
                                 if (IDNo != null && !IDNo.trim().equals("")) {
-                                    xmlw.writeStartElement("titlStmt");
+
                                     xmlw.writeStartElement("IDNo");
                                     if (IDType != null && !IDType.trim().equals("")) {
-                                        xmlw.writeAttribute("agency", IDType );
+                                        xmlw.writeAttribute("agency", IDType);
                                     }
                                     xmlw.writeCharacters(IDNo);
                                     xmlw.writeEndElement(); //IDNo
-                                    xmlw.writeEndElement(); // titlStmt
                                 }
+                                xmlw.writeEndElement(); // titlStmt
+
 
                                 writeFullElement(xmlw,"biblCit",citation);
                                 xmlw.writeEndElement(); //citation
@@ -1181,33 +1296,34 @@ private static void writeSeriesElement(XMLStreamWriter xmlw, DatasetVersionDTO d
         for (Map.Entry<String, MetadataBlockDTO> entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
             String key = entry.getKey();
             MetadataBlockDTO value = entry.getValue();
-            if ("citation".equals(key)) {
+            if ("citation".equals(key)) {               
                 for (FieldDTO fieldDTO : value.getFields()) {
                     if (DatasetFieldConstant.series.equals(fieldDTO.getTypeName())) {
-                        xmlw.writeStartElement("serStmt");                        
                         String seriesName = "";
                         String seriesInformation = "";
-                        Set<FieldDTO> foo = fieldDTO.getSingleCompound();
+                        for (HashSet<FieldDTO> foo : fieldDTO.getMultipleCompound()) {
+                            xmlw.writeStartElement("serStmt");
                             for (Iterator<FieldDTO> iterator = foo.iterator(); iterator.hasNext();) {
                                 FieldDTO next = iterator.next();
                                 if (DatasetFieldConstant.seriesName.equals(next.getTypeName())) {
-                                    seriesName =  next.getSinglePrimitive();
+                                    seriesName = next.getSinglePrimitive();
                                 }
                                 if (DatasetFieldConstant.seriesInformation.equals(next.getTypeName())) {
-                                    seriesInformation =  next.getSinglePrimitive();
+                                    seriesInformation = next.getSinglePrimitive();
                                 }
                             }
-                            if (!seriesName.isEmpty()){
-                                xmlw.writeStartElement("serName"); 
+                            if (!seriesName.isEmpty()) {
+                                xmlw.writeStartElement("serName");
                                 xmlw.writeCharacters(seriesName);
-                                xmlw.writeEndElement(); //grantno
+                                xmlw.writeEndElement(); //serName
                             }
-                            if (!seriesInformation.isEmpty()){
-                                xmlw.writeStartElement("serInfo"); 
+                            if (!seriesInformation.isEmpty()) {
+                                xmlw.writeStartElement("serInfo");
                                 xmlw.writeCharacters(seriesInformation);
-                                xmlw.writeEndElement(); //grantno
+                                xmlw.writeEndElement(); //serInfo
                             }
-                        xmlw.writeEndElement(); //serStmt
+                            xmlw.writeEndElement(); //serStmt
+                        }
                     }
                 }
             }
@@ -1234,17 +1350,18 @@ private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersio
                                 actualSize = next.getSinglePrimitive();
                             }
                         }
-
-                        if (!sizeFormula.isEmpty()) {
-                            xmlw.writeStartElement("sampleSizeFormula");
-                            xmlw.writeCharacters(sizeFormula);
-                            xmlw.writeEndElement(); //sampleSizeFormula
-                        }
+                        /* <sampleSize> must come before <sampleSizeFormula>! -L.A. */
                         if (!actualSize.isEmpty()) {
                             xmlw.writeStartElement("sampleSize");
                             xmlw.writeCharacters(actualSize);
                             xmlw.writeEndElement(); //sampleSize
                         }
+                        if (!sizeFormula.isEmpty()) {
+                            xmlw.writeStartElement("sampleSizeFormula");
+                            xmlw.writeCharacters(sizeFormula);
+                            xmlw.writeEndElement(); //sampleSizeFormula
+                        }
+                        
                         xmlw.writeEndElement(); // targetSampleSize
                     }
                 }
@@ -1356,8 +1473,8 @@ private static void createOtherMatsFromFileMetadatas(XMLStreamWriter xmlw, List<
                 writeAttribute(xmlw, "ID", "f" + fileMetadata.getDataFile().getId());
                 String dfIdentifier = fileMetadata.getDataFile().getIdentifier();
                 if (dfIdentifier != null && !dfIdentifier.isEmpty()){
-                    GlobalId globalId = new GlobalId(fileMetadata.getDataFile());
-                    writeAttribute(xmlw, "URI",  globalId.toURL().toString()); 
+                    GlobalId globalId = fileMetadata.getDataFile().getGlobalId();
+                    writeAttribute(xmlw, "URI",  globalId.asURL()); 
                 }  else {
                     writeAttribute(xmlw, "URI", dataverseUrl + "/api/access/datafile/" + fileMetadata.getDataFile().getId()); 
                 }
@@ -1432,7 +1549,15 @@ private static List<String> dto2PrimitiveList(DatasetVersionDTO datasetVersionDT
             MetadataBlockDTO value = entry.getValue();
             for (FieldDTO fieldDTO : value.getFields()) {
                 if (datasetFieldTypeName.equals(fieldDTO.getTypeName())) {
-                    return fieldDTO.getMultiplePrimitive();
+                    // This hack is here to make sure the export does not blow 
+                    // up on an instance that upgraded to a Dataverse version
+                    // where a certain primitive has been made multiple, but has
+                    // not yet update the block. 
+                    if (fieldDTO.getMultiple() != null && fieldDTO.getMultiple()) {
+                        return fieldDTO.getMultiplePrimitive();
+                    } else {
+                        return Arrays.asList(fieldDTO.getSinglePrimitive());
+                    }
                 }
             }
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
index 34cb7a4e138..334b18f4601 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
@@ -1,13 +1,8 @@
 package edu.harvard.iq.dataverse.export.openaire;
 
 import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.logging.Logger;
-import java.util.List;
 
 import javax.json.JsonObject;
 import javax.xml.stream.XMLOutputFactory;
@@ -18,13 +13,17 @@
 
 import com.google.gson.Gson;
 
+import edu.harvard.iq.dataverse.DOIServiceBean;
 import edu.harvard.iq.dataverse.DatasetFieldConstant;
 import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.HandlenetServiceBean;
 import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
 import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
 import edu.harvard.iq.dataverse.api.dto.DatasetVersionDTO;
 import edu.harvard.iq.dataverse.api.dto.FieldDTO;
 import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO;
+import edu.harvard.iq.dataverse.util.PersonOrOrgUtil;
+import edu.harvard.iq.dataverse.pidproviders.PidUtil;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -71,7 +70,7 @@ private static void createOpenAire(XMLStreamWriter xmlw, DatasetDTO datasetDto)
         String persistentAgency = datasetDto.getProtocol();
         String persistentAuthority = datasetDto.getAuthority();
         String persistentId = datasetDto.getIdentifier();
-        GlobalId globalId = new GlobalId(persistentAgency, persistentAuthority, persistentId);
+        GlobalId globalId = PidUtil.parseAsGlobalID(persistentAgency, persistentAuthority, persistentId);
 
         // The sequence is revied using sample:
         // https://schema.datacite.org/meta/kernel-4.0/example/datacite-example-full-v4.0.xml
@@ -83,7 +82,7 @@ private static void createOpenAire(XMLStreamWriter xmlw, DatasetDTO datasetDto)
         String language = null;
 
         // 1, Identifier (with mandatory type sub-property) (M)
-        writeIdentifierElement(xmlw, globalId.toURL().toString(), language);
+        writeIdentifierElement(xmlw, globalId.asURL(), language);
 
         // 2, Creator (with optional given name, family name, 
         //      name identifier and affiliation sub-properties) (M)
@@ -191,10 +190,10 @@ public static void writeIdentifierElement(XMLStreamWriter xmlw, String identifie
         if (StringUtils.isNotBlank(identifier)) {
             Map<String, String> identifier_map = new HashMap<String, String>();
 
-            if (StringUtils.containsIgnoreCase(identifier, GlobalId.DOI_RESOLVER_URL)) {
+            if (StringUtils.containsIgnoreCase(identifier, DOIServiceBean.DOI_RESOLVER_URL)) {
                 identifier_map.put("identifierType", "DOI");
                 identifier = StringUtils.substring(identifier, identifier.indexOf("10."));
-            } else if (StringUtils.containsIgnoreCase(identifier, GlobalId.HDL_RESOLVER_URL)) {
+            } else if (StringUtils.containsIgnoreCase(identifier, HandlenetServiceBean.HDL_RESOLVER_URL)) {
                 identifier_map.put("identifierType", "Handle");
                 if (StringUtils.contains(identifier, "http")) {
                     identifier = identifier.replace(identifier.substring(0, identifier.indexOf("/") + 2), "");
@@ -250,72 +249,26 @@ public static void writeCreatorsElement(XMLStreamWriter xmlw, DatasetVersionDTO
                             if (StringUtils.isNotBlank(creatorName)) {
                                 creator_check = writeOpenTag(xmlw, "creators", creator_check);
                                 xmlw.writeStartElement("creator"); // <creator>
-
-                                boolean nameType_check = false;
+                                
                                 Map<String, String> creator_map = new HashMap<String, String>();
-                                if ((StringUtils.containsIgnoreCase(nameIdentifierScheme, "orcid"))) {
+                                JsonObject creatorObj = PersonOrOrgUtil.getPersonOrOrganization(creatorName, false,
+                                        StringUtils.containsIgnoreCase(nameIdentifierScheme, "orcid"));
+
+                                // creatorName=<FamilyName>, <FirstName>
+                                if (creatorObj.getBoolean("isPerson")) {
                                     creator_map.put("nameType", "Personal");
-                                    nameType_check = true;
-                                }
-                                // ToDo - the algorithm to determine if this is a Person or Organization here
-                                // has been abstracted into a separate
-                                // edu.harvard.iq.dataverse.util.PersonOrOrgUtil class that could be used here
-                                // to avoid duplication/variants of the algorithm
-                                creatorName = Cleanup.normalize(creatorName);
-                                // Datacite algorithm, https://github.com/IQSS/dataverse/issues/2243#issuecomment-358615313
-                                if (creatorName.contains(",")) {
-                                    String givenName = FirstNames.getInstance().getFirstName(creatorName);
-                                    boolean isOrganization = Organizations.getInstance().isOrganization(creatorName);
-
-                                    // creatorName=<FamilyName>, <FirstName>
-                                    if (givenName != null && !isOrganization) {
-                                        // givenName ok
-                                        creator_map.put("nameType", "Personal");
-                                        nameType_check = true;
-                                    } else if (isOrganization) {
-                                        creator_map.put("nameType", "Organizational");
-                                        nameType_check = false;
-                                    }
-                                    writeFullElement(xmlw, null, "creatorName", creator_map, creatorName, language);
-
-                                    if ((nameType_check) && (!creatorName.replaceFirst(",", "").contains(","))) {
-                                        // creatorName=<FamilyName>, <FirstName>
-                                        String[] fullName = creatorName.split(", ");
-                                        if (fullName.length == 2) {
-                                            givenName = fullName[1];
-                                            String familyName = fullName[0];
-
-                                            writeFullElement(xmlw, null, "givenName", null, givenName, language);
-                                            writeFullElement(xmlw, null, "familyName", null, familyName, language);
-                                        } else {
-                                            // It's possible to get here if "Smith," is entered as an author name.
-                                            logger.info("Unable to write givenName and familyName based on creatorName '" + creatorName + "'.");
-                                        }
-                                    }
                                 } else {
-                                    String givenName = FirstNames.getInstance().getFirstName(creatorName);
-                                    boolean isOrganization = Organizations.getInstance().isOrganization(creatorName);
-                                    
-                                    if (givenName != null && !isOrganization) {
-                                        // givenName ok, creatorName=<FirstName> <FamilyName>
-                                        creator_map.put("nameType", "Personal");
-                                        nameType_check = true;
-                                        writeFullElement(xmlw, null, "creatorName", creator_map, creatorName, language);
-
-                                        String familyName = "";
-                                        if (givenName.length() + 1 < creatorName.length()) {
-                                            familyName = creatorName.substring(givenName.length() + 1);
-                                        }
-
-                                        writeFullElement(xmlw, null, "givenName", null, givenName, language);
-                                        writeFullElement(xmlw, null, "familyName", null, familyName, language);
-                                    } else {
-                                        // default
-                                        if (isOrganization) {
-                                            creator_map.put("nameType", "Organizational");
-                                        }
-                                        writeFullElement(xmlw, null, "creatorName", creator_map, creatorName, language);
-                                    }
+                                    creator_map.put("nameType", "Organizational");
+                                }
+                                writeFullElement(xmlw, null, "creatorName", creator_map,
+                                        creatorObj.getString("fullName"), language);
+                                if (creatorObj.containsKey("givenName")) {
+                                    writeFullElement(xmlw, null, "givenName", null, creatorObj.getString("givenName"),
+                                            language);
+                                }
+                                if (creatorObj.containsKey("familyName")) {
+                                    writeFullElement(xmlw, null, "familyName", null, creatorObj.getString("familyName"),
+                                            language);
                                 }
 
                                 if (StringUtils.isNotBlank(nameIdentifier)) {
@@ -369,10 +322,34 @@ public static void writeTitlesElement(XMLStreamWriter xmlw, DatasetVersionDTO da
         String subtitle = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.subTitle);
         title_check = writeTitleElement(xmlw, "Subtitle", subtitle, title_check, language);
 
+        //String alternativeTitle = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.alternativeTitle);
+        //title_check = writeTitleElement(xmlw, "AlternativeTitle", alternativeTitle, title_check, language);
         title_check = writeMultipleTitleElement(xmlw, "AlternativeTitle", datasetVersionDTO, "citation", title_check, language);
+        writeEndTag(xmlw, title_check);
+    }
 
+    private static boolean writeMultipleTitleElement(XMLStreamWriter xmlw, String titleType, DatasetVersionDTO datasetVersionDTO, String metadataBlockName, boolean title_check, String language) throws XMLStreamException {
+        MetadataBlockDTO block = datasetVersionDTO.getMetadataBlocks().get(metadataBlockName);
+        if (block != null) {
+            logger.info("Block is not empty");
+            List<FieldDTO> fieldsBlock =  block.getFields();
+            if (fieldsBlock != null) {
+                for (FieldDTO fieldDTO : fieldsBlock) {
+                    logger.info(titleType + " " + fieldDTO.getTypeName());
+                    if (titleType.toLowerCase().equals(fieldDTO.getTypeName().toLowerCase())) {
+                        logger.info("Found Alt title");
+                        List<String> fields = fieldDTO.getMultiplePrimitive();
+                        for (String value : fields) {
+                            if (!writeTitleElement(xmlw, titleType, value, title_check, language))
+                                title_check = false;
+                        }
+                        break;
+                    }
+                }
+            }
+        }
 
-        writeEndTag(xmlw, title_check);
+        return title_check;
     }
 
     /**
@@ -405,33 +382,6 @@ private static boolean writeTitleElement(XMLStreamWriter xmlw, String titleType,
         }
         return title_check;
     }
-    
-    private static boolean writeMultipleTitleElement(XMLStreamWriter xmlw, String titleType, DatasetVersionDTO datasetVersionDTO, String metadataBlockName, boolean title_check, String language) throws XMLStreamException {
-        MetadataBlockDTO block = datasetVersionDTO.getMetadataBlocks().get(metadataBlockName);
-        if (block != null) {
-            logger.info("Block is not empty");
-            List<FieldDTO> fieldsBlock =  block.getFields();
-            if (fieldsBlock != null) {
-                for (FieldDTO fieldDTO : fieldsBlock) {
-                    logger.info(titleType + " " + fieldDTO.getTypeName());
-                    if (titleType.toLowerCase().equals(fieldDTO.getTypeName().toLowerCase())) {
-                        logger.info("Found Alt title");
-                        List<String> fields = fieldDTO.getMultiplePrimitive();
-                        for (String value : fields) {
-                            if (!writeTitleElement(xmlw, titleType, value, title_check, language))
-                                title_check = false;
-                        }
-                        break;
-                    }
-                }
-            }
-        }
-
-        return title_check;
-    }
-
-
-
 
     /**
      * 5, PublicationYear (M)
@@ -737,61 +687,23 @@ public static void writeContributorElement(XMLStreamWriter xmlw, String contribu
         boolean nameType_check = false;
         Map<String, String> contributor_map = new HashMap<String, String>();
 
-        // ToDo - the algorithm to determine if this is a Person or Organization here
-        // has been abstracted into a separate
-        // edu.harvard.iq.dataverse.util.PersonOrOrgUtil class that could be used here
-        // to avoid duplication/variants of the algorithm
+        JsonObject contributorObj = PersonOrOrgUtil.getPersonOrOrganization(contributorName,
+                ("ContactPerson".equals(contributorType) && !isValidEmailAddress(contributorName)), false);
 
-        contributorName = Cleanup.normalize(contributorName);
-        // Datacite algorithm, https://github.com/IQSS/dataverse/issues/2243#issuecomment-358615313
-        if (contributorName.contains(",")) {
-            String givenName = FirstNames.getInstance().getFirstName(contributorName);
-            boolean isOrganization = Organizations.getInstance().isOrganization(contributorName);
-
-            // contributorName=<FamilyName>, <FirstName>
-            if (givenName != null && !isOrganization) {
-                // givenName ok
+        if (contributorObj.getBoolean("isPerson")) {
+            if(contributorObj.containsKey("givenName")) {
                 contributor_map.put("nameType", "Personal");
-                nameType_check = true;
-                // re: the above toDo - the ("ContactPerson".equals(contributorType) &&
-                // !isValidEmailAddress(contributorName)) clause in the next line could/should
-                // be sent as the OrgIfTied boolean parameter
-            } else if (isOrganization || ("ContactPerson".equals(contributorType) && !isValidEmailAddress(contributorName))) {
-                contributor_map.put("nameType", "Organizational");
-            }
-            writeFullElement(xmlw, null, "contributorName", contributor_map, contributorName, language);
-
-            if ((nameType_check) && (!contributorName.replaceFirst(",", "").contains(","))) {
-                // contributorName=<FamilyName>, <FirstName>
-                String[] fullName = contributorName.split(", ");
-                givenName = fullName[1];
-                String familyName = fullName[0];
-
-                writeFullElement(xmlw, null, "givenName", null, givenName, language);
-                writeFullElement(xmlw, null, "familyName", null, familyName, language);
             }
         } else {
-            String givenName = FirstNames.getInstance().getFirstName(contributorName);
-            boolean isOrganization = Organizations.getInstance().isOrganization(contributorName);
-
-            if (givenName != null && !isOrganization) {
-                contributor_map.put("nameType", "Personal");
-                writeFullElement(xmlw, null, "contributorName", contributor_map, contributorName, language);
-
-                String familyName = "";
-                if (givenName.length() + 1 < contributorName.length()) {
-                    familyName = contributorName.substring(givenName.length() + 1);
-                }
+            contributor_map.put("nameType", "Organizational");
+        }
+        writeFullElement(xmlw, null, "contributorName", contributor_map, contributorName, language);
 
-                writeFullElement(xmlw, null, "givenName", null, givenName, language);
-                writeFullElement(xmlw, null, "familyName", null, familyName, language);
-            } else {
-                // default
-                if (isOrganization || ("ContactPerson".equals(contributorType) && !isValidEmailAddress(contributorName))) {
-                    contributor_map.put("nameType", "Organizational");
-                }
-                writeFullElement(xmlw, null, "contributorName", contributor_map, contributorName, language);
-            }
+        if (contributorObj.containsKey("givenName")) {
+            writeFullElement(xmlw, null, "givenName", null, contributorObj.getString("givenName"), language);
+        }
+        if (contributorObj.containsKey("familyName")) {
+            writeFullElement(xmlw, null, "familyName", null, contributorObj.getString("familyName"), language);
         }
 
         if (StringUtils.isNotBlank(contributorAffiliation)) {
@@ -1291,26 +1203,17 @@ public static void writeDescriptionsElement(XMLStreamWriter xmlw, DatasetVersion
                     if (DatasetFieldConstant.series.equals(fieldDTO.getTypeName())) {
                         // String seriesName = null;
                         String seriesInformation = null;
-
-                        Set<FieldDTO> fieldDTOs = fieldDTO.getSingleCompound();
-                        for (Iterator<FieldDTO> iterator = fieldDTOs.iterator(); iterator.hasNext();) {
-                            FieldDTO next = iterator.next();
-                            /*if (DatasetFieldConstant.seriesName.equals(next.getTypeName())) {
-                                seriesName =  next.getSinglePrimitive();
-                            }*/
-                            if (DatasetFieldConstant.seriesInformation.equals(next.getTypeName())) {
-                                seriesInformation = next.getSinglePrimitive();
+                        for (HashSet<FieldDTO> fieldDTOs : fieldDTO.getMultipleCompound()) {
+                            for (Iterator<FieldDTO> iterator = fieldDTOs.iterator(); iterator.hasNext();) {
+                                FieldDTO next = iterator.next();
+                                if (DatasetFieldConstant.seriesInformation.equals(next.getTypeName())) {
+                                    seriesInformation = next.getSinglePrimitive();
+                                }
+                            }
+                            if (StringUtils.isNotBlank(seriesInformation)) {
+                                description_check = writeOpenTag(xmlw, "descriptions", description_check);
+                                writeDescriptionElement(xmlw, "SeriesInformation", seriesInformation, language);
                             }
-                        }
-
-                        /*if (StringUtils.isNotBlank(seriesName)){
-                        	contributor_check = writeOpenTag(xmlw, "descriptions", description_check);
-                        	
-                        	writeDescriptionElement(xmlw, "SeriesInformation", seriesName);
-                        }*/
-                        if (StringUtils.isNotBlank(seriesInformation)) {
-                            description_check = writeOpenTag(xmlw, "descriptions", description_check);
-                            writeDescriptionElement(xmlw, "SeriesInformation", seriesInformation, language);
                         }
                     }
                 }

From 3bb7cbfae2072f7bb4f5b11567cd0b02c0b4bb02 Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Thu, 11 May 2023 17:27:05 -0400
Subject: [PATCH 056/922] add prodPlac

---
 .../iq/dataverse/api/imports/ImportDDIServiceBean.java    | 8 +++++---
 .../harvard/iq/dataverse/export/ddi/DdiExportUtil.java    | 6 +++++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index bafd7267acb..ae98e7e76ea 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -1339,6 +1339,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
         List<HashSet<FieldDTO>> producers = new ArrayList<>();
         List<HashSet<FieldDTO>> grants = new ArrayList<>();
         List<HashSet<FieldDTO>> software = new ArrayList<>();
+        List<String> prodPlac = new ArrayList<>();
 
         for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
             if (event == XMLStreamConstants.START_ELEMENT) {
@@ -1354,9 +1355,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
                 } else if (xmlr.getLocalName().equals("prodDate")) {
                     citation.getFields().add(FieldDTO.createPrimitiveFieldDTO("productionDate", parseDate(xmlr, "prodDate")));
                 } else if (xmlr.getLocalName().equals("prodPlac")) {
-                    List<String> prodPlac = new ArrayList<>();
-                    prodPlac.add(parseText(xmlr, "prodPlac"));
-                    citation.getFields().add(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.productionPlace, prodPlac));
+                    prodPlac.add(parseText(xmlr));
                 } else if (xmlr.getLocalName().equals("software")) {
                     HashSet<FieldDTO> set = new HashSet<>();
                     addToSet(set,"softwareVersion", xmlr.getAttributeValue(null, "version"));
@@ -1389,6 +1388,9 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th
                     if (producers.size()>0) {
                         citation.getFields().add(FieldDTO.createMultipleCompoundFieldDTO("producer", producers));
                     }
+                    if (prodPlac.size() > 0) {
+                        citation.getFields().add(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.productionPlace, prodPlac));
+                    }
                     return;
                 }
             }
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index a647c2a6f2a..cd9311ec518 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -1006,7 +1006,11 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT
         // productionPlace was made multiple as of 5.14:
         // (a quick backward compatibility check was added to dto2PrimitiveList(),
         // see the method for details)
-        writeFullElementList(xmlw, "prodPlac", dto2PrimitiveList(version, DatasetFieldConstant.productionPlace));        
+
+        FieldDTO  prodPlac = dto2FieldDTO( version, DatasetFieldConstant.productionPlace, "citation"  );
+        if (prodPlac != null) {
+            writeMultipleElement(xmlw, "prodPlac", prodPlac, null);
+        }
         writeSoftwareElement(xmlw, version);
   
         writeGrantElement(xmlw, version);

From cd47f9389e881866a45792b5d9cf9b286c5d7fa7 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 09:30:23 +0200
Subject: [PATCH 057/922] chore(deps): update Nimbus SDK to 10.9.1

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 49443f62453..8764e4f493d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -385,7 +385,7 @@
         <dependency>
             <groupId>com.nimbusds</groupId>
             <artifactId>oauth2-oidc-sdk</artifactId>
-            <version>10.7.1</version>
+            <version>10.9.1</version>
         </dependency>
         <!-- New and Improved GDCC XOAI library! --> 
         <dependency>

From 94598bd66de2ee5a535cca6aab63b020ad95f65b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 10:04:39 +0200
Subject: [PATCH 058/922] feat(auth): pass OAuth2/OIDC state parameter to
 provider

When the client is returning from the provider to us, carrying along
the authorization code we need to retrieve user details, we also
receive again the state. The state was generated and sent by us,
and will not be altered by the provider, which makes it perfect to
identify the original request we built before sending the client to
the provider.

Passing this state to the provider enables the provider to reuse this
information. This is crucial to enable PKCE support, as we need to
remember which secret code we sent to the provider - otherwise we
will not be able to verify the authz code.

Tests have been adapted.
---
 .../oauth2/AbstractOAuth2AuthenticationProvider.java  |  3 ++-
 .../providers/oauth2/OAuth2LoginBackingBean.java      |  2 +-
 .../providers/oauth2/OAuth2LoginBackingBeanTest.java  | 11 ++++++++---
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java
index 01139cd2e27..373a295487d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/AbstractOAuth2AuthenticationProvider.java
@@ -139,6 +139,7 @@ public OAuth20Service getService(String callbackUrl) {
      * Receive user data from OAuth2 provider after authn/z has been successfull. (Callback view uses this)
      * Request a token and access the resource, parse output and return user details.
      * @param code The authz code sent from the provider
+     * @param state The state which was communicated between us and the provider, identifying the exact request
      * @param redirectUrl The redirect URL (some providers require this when fetching the access token, e. g. Google)
      * @return A user record containing all user details accessible for us
      * @throws IOException Thrown when communication with the provider fails
@@ -146,7 +147,7 @@ public OAuth20Service getService(String callbackUrl) {
      * @throws InterruptedException Thrown when the requests thread is failing
      * @throws ExecutionException Thrown when the requests thread is failing
      */
-    public OAuth2UserRecord getUserRecord(String code, String redirectUrl)
+    public OAuth2UserRecord getUserRecord(String code, String state, String redirectUrl)
         throws IOException, OAuth2Exception, InterruptedException, ExecutionException {
         
         OAuth20Service service = getService(redirectUrl);
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java
index c5be41a014a..7b52f2e9b16 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java
@@ -100,7 +100,7 @@ public void exchangeCodeForToken() throws IOException {
 
             if (oIdp.isPresent() && code.isPresent()) {
                 AbstractOAuth2AuthenticationProvider idp = oIdp.get();
-                oauthUser = idp.getUserRecord(code.get(), systemConfig.getOAuth2CallbackUrl());
+                oauthUser = idp.getUserRecord(code.get(), req.getParameter("state"), systemConfig.getOAuth2CallbackUrl());
                 
                 // Throw an error if this authentication method is disabled:
                 // (it's not clear if it's possible at all, for somebody to get here with 
diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBeanTest.java
index 80249cc89e8..0c54c050d79 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBeanTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBeanTest.java
@@ -105,6 +105,7 @@ class ecft {
         @Mock DataverseSession session;
         @Mock OAuth2TokenDataServiceBean oauth2Tokens;
         Optional<String> redirect = Optional.of("/hellotest");
+        String state;
         
         @BeforeEach
         void setUp() throws IOException {
@@ -118,7 +119,11 @@ void setUp() throws IOException {
             when(externalContextMock.getRequest()).thenReturn(requestMock);
             lenient().when(externalContextMock.getFlash()).thenReturn(flashMock);
             lenient().when(requestMock.getReader()).thenReturn(reader);
-            doReturn(loginBackingBean.createState(testIdp, this.redirect)).when(requestMock).getParameter("state");
+            
+            // Save the state as we need it for injection (necessary because of PKCE support)
+            state = loginBackingBean.createState(testIdp, this.redirect);
+            doReturn(state).when(requestMock).getParameter("state");
+            
             // travel in time at least 10 milliseconds (remote calls & redirects are much likely longer)
             // (if not doing this tests become flaky on fast machinas)
             loginBackingBean.clock = Clock.offset(constantClock, Duration.ofMillis(10));
@@ -140,7 +145,7 @@ void newUser() throws Exception {
             // fake the code received from the provider
             when(requestMock.getParameter("code")).thenReturn(code);
             // let's deep-fake the result of getUserRecord()
-            doReturn(userRecord).when(testIdp).getUserRecord(code, null);
+            doReturn(userRecord).when(testIdp).getUserRecord(code, state, null);
     
             // WHEN (& then)
             // capture the redirect target from the faces context
@@ -168,7 +173,7 @@ void existingUser() throws Exception {
             // fake the code received from the provider
             when(requestMock.getParameter("code")).thenReturn(code);
             // let's deep-fake the result of getUserRecord()
-            doReturn(userRecord).when(testIdp).getUserRecord(code, null);
+            doReturn(userRecord).when(testIdp).getUserRecord(code, state, null);
             doReturn(tokenData).when(userRecord).getTokenData();
             // also fake the result of the lookup in the auth service
             doReturn(userIdentifier).when(userRecord).getUserRecordIdentifier();

From 5fbee2e067722e7ff649dd30ae3e8afa90851958 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 10:05:03 +0200
Subject: [PATCH 059/922] feat(auth): add OIDC PKCE settings to JvmSettings

---
 .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index dc9267805e6..ba90d895ae9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -121,6 +121,9 @@ public enum JvmSettings {
     OIDC_AUTH_SERVER_URL(SCOPE_OIDC, "auth-server-url"),
     OIDC_CLIENT_ID(SCOPE_OIDC, "client-id"),
     OIDC_CLIENT_SECRET(SCOPE_OIDC, "client-secret"),
+    SCOPE_OIDC_PKCE(SCOPE_OIDC, "pkce"),
+    OIDC_PKCE_ENABLED(SCOPE_OIDC_PKCE, "enabled"),
+    OIDC_PKCE_METHOD(SCOPE_OIDC_PKCE, "method"),
     
     ;
     

From c0d21cc9f935b3dc43653baa3d132681ff94c1c0 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 10:07:35 +0200
Subject: [PATCH 060/922] feat(auth): enable PKCE usage in OIDC provider

- Adding PKCE parameters to constructor
- Adding a hashmap to cache the code verifiers mapped by the unique state we generate
- Enabling the actual workflow of PKCE
---
 .../oauth2/oidc/OIDCAuthProvider.java         | 38 ++++++++++++++++---
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
index 4b6c575cfaf..91d552618ef 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
@@ -18,6 +18,8 @@
 import com.nimbusds.oauth2.sdk.id.ClientID;
 import com.nimbusds.oauth2.sdk.id.Issuer;
 import com.nimbusds.oauth2.sdk.id.State;
+import com.nimbusds.oauth2.sdk.pkce.CodeChallengeMethod;
+import com.nimbusds.oauth2.sdk.pkce.CodeVerifier;
 import com.nimbusds.oauth2.sdk.token.BearerAccessToken;
 import com.nimbusds.openid.connect.sdk.AuthenticationRequest;
 import com.nimbusds.openid.connect.sdk.Nonce;
@@ -39,7 +41,9 @@
 import java.net.URI;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.logging.Logger;
 
@@ -57,12 +61,25 @@ public class OIDCAuthProvider extends AbstractOAuth2AuthenticationProvider {
     final Issuer issuer;
     final ClientAuthentication clientAuth;
     final OIDCProviderMetadata idpMetadata;
+    final boolean pkceEnabled;
+    final CodeChallengeMethod pkceMethod;
     
-    public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEndpointURL) throws AuthorizationSetupException {
+    /**
+     * Using PKCE, we create and send a special {@link CodeVerifier}. This contains a secret
+     * we need again when verifying the response by the provider, thus the cache.
+     */
+    private final Map<String,CodeVerifier> verifierCache = new ConcurrentHashMap<>();
+    
+    public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEndpointURL,
+                            boolean pkceEnabled, String pkceMethod) throws AuthorizationSetupException {
         this.clientSecret = aClientSecret; // nedded for state creation
         this.clientAuth = new ClientSecretBasic(new ClientID(aClientId), new Secret(aClientSecret));
         this.issuer = new Issuer(issuerEndpointURL);
+        
         this.idpMetadata = getMetadata();
+        
+        this.pkceEnabled = pkceEnabled;
+        this.pkceMethod = CodeChallengeMethod.parse(pkceMethod);
     }
     
     /**
@@ -147,6 +164,7 @@ public String buildAuthzUrl(String state, String callbackUrl) {
         State stateObject = new State(state);
         URI callback = URI.create(callbackUrl);
         Nonce nonce = new Nonce();
+        CodeVerifier pkceVerifier = pkceEnabled ? new CodeVerifier() : null;
         
         AuthenticationRequest req = new AuthenticationRequest.Builder(new ResponseType("code"),
                                                                       Scope.parse(this.scope),
@@ -154,9 +172,15 @@ public String buildAuthzUrl(String state, String callbackUrl) {
                                                                       callback)
             .endpointURI(idpMetadata.getAuthorizationEndpointURI())
             .state(stateObject)
+            // Called method is nullsafe - will disable sending a PKCE challenge in case the verifier is not present
+            .codeChallenge(pkceVerifier, pkceMethod)
             .nonce(nonce)
             .build();
         
+        // Cache the PKCE verifier, as we need the secret in it for verification later again, after the client sends us
+        // the auth code! We use the state to cache the verifier, as the state is unique per authentication event.
+        this.verifierCache.put(state, pkceVerifier);
+        
         return req.toURI().toString();
     }
     
@@ -172,10 +196,14 @@ public String buildAuthzUrl(String state, String callbackUrl) {
      * @throws ExecutionException Thrown when the requests thread is failing
      */
     @Override
-    public OAuth2UserRecord getUserRecord(String code, String redirectUrl)
-        throws IOException, OAuth2Exception, InterruptedException, ExecutionException {
-        // Create grant object
-        AuthorizationGrant codeGrant = new AuthorizationCodeGrant(new AuthorizationCode(code), URI.create(redirectUrl));
+    public OAuth2UserRecord getUserRecord(String code, String state, String redirectUrl) throws IOException, OAuth2Exception {
+        // Retrieve the verifier from the cache and clear from the cache. If not found, will be null.
+        // Will be sent to token endpoint for verification, so if required but missing, will lead to exception.
+        CodeVerifier verifier = verifierCache.remove(state);
+        
+        // Create grant object - again, this is null-safe for the verifier
+        AuthorizationGrant codeGrant = new AuthorizationCodeGrant(
+            new AuthorizationCode(code), URI.create(redirectUrl), verifier);
     
         // Get Access Token first
         Optional<BearerAccessToken> accessToken = getAccessToken(codeGrant);

From ef4d192156bd7efd6e83226c57dd7deea545a6e2 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 10:08:37 +0200
Subject: [PATCH 061/922] feat(auth): extend OIDC provider factory to
 understand PKCE parameters

To enable backward compatibility, default to disabled and method S256.
---
 .../oauth2/oidc/OIDCAuthenticationProviderFactory.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
index 89cf1cb986d..3f8c18d0567 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactory.java
@@ -41,7 +41,9 @@ public AuthenticationProvider buildProvider( AuthenticationProviderRow aRow ) th
         OIDCAuthProvider oidc = new OIDCAuthProvider(
             factoryData.get("clientId"),
             factoryData.get("clientSecret"),
-            factoryData.get("issuer")
+            factoryData.get("issuer"),
+            Boolean.parseBoolean(factoryData.getOrDefault("pkceEnabled", "false")),
+            factoryData.getOrDefault("pkceMethod", "S256")
         );
         
         oidc.setId(aRow.getId());
@@ -60,7 +62,9 @@ public static AuthenticationProvider buildFromSettings() throws AuthorizationSet
         OIDCAuthProvider oidc = new OIDCAuthProvider(
             JvmSettings.OIDC_CLIENT_ID.lookup(),
             JvmSettings.OIDC_CLIENT_SECRET.lookup(),
-            JvmSettings.OIDC_AUTH_SERVER_URL.lookup()
+            JvmSettings.OIDC_AUTH_SERVER_URL.lookup(),
+            JvmSettings.OIDC_PKCE_ENABLED.lookupOptional(Boolean.class).orElse(false),
+            JvmSettings.OIDC_PKCE_METHOD.lookupOptional().orElse("S256")
         );
         
         oidc.setId("oidc-mpconfig");

From 37bcc3a69930879810c7a7eb87f465219a00a24d Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 10:28:32 +0200
Subject: [PATCH 062/922] doc(auth): add OIDC PKCE configuration

---
 .../source/installation/oidc.rst              | 34 ++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index fbcbd3eb4ad..9848d73b189 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -69,9 +69,32 @@ After adding a provider, the Log In page will by default show the "builtin" prov
    In contrast to our :doc:`oauth2`, you can use multiple providers by creating distinct configurations enabled by
    the same technology and without modifying the Dataverse Software code base (standards for the win!).
 
+
+.. _oidc-pkce:
+
+Enabling PKCE Security
+^^^^^^^^^^^^^^^^^^^^^^
+
+Many providers these days support or even require the usage of `PKCE <https://oauth.net/2/pkce/>`_ to safeguard against
+some attacks and enable public clients that cannot have a secure secret to still use OpenID Connect (or OAuth2).
+
+The Dataverse built OIDC client can be enabled to use PKCE and which method to use when creating the code challenge.
+See also `this explanation of the flow <https://auth0.com/docs/get-started/authentication-and-authorization-flow/authorization-code-flow-with-proof-key-for-code-exchange-pkce>`_
+for details on how this works.
+
+As we are using the `Nimbus SDK <https://connect2id.com/products/nimbus-oauth-openid-connect-sdk>`_ as our client
+library, we support the standard ``PLAIN`` and ``S256`` code challenge methods. "SHA-256 method" is the default
+as recommend in `RFC7636 <https://datatracker.ietf.org/doc/html/rfc7636#section-4.2>`_. If your provider needs some
+other method (unlikely), please open an issue.
+
+The provisioning sections below contain in the example the parameters you may use to configure PKCE.
+
 Provision via REST API
 ^^^^^^^^^^^^^^^^^^^^^^
 
+Note: you may omit the PKCE related settings from ``factoryData`` below if you don't plan on using PKCE - default is
+disabled.
+
 Please create a ``my-oidc-provider.json`` file like this, replacing every ``<...>`` with your values:
 
 .. code-block:: json
@@ -81,7 +104,7 @@ Please create a ``my-oidc-provider.json`` file like this, replacing every ``<...
         "factoryAlias":"oidc",
         "title":"<a title - shown in UI>",
         "subtitle":"<a subtitle - currently unused in UI>",
-        "factoryData":"type: oidc | issuer: <issuer url> | clientId: <client id> | clientSecret: <client secret>",
+        "factoryData":"type: oidc | issuer: <issuer url> | clientId: <client id> | clientSecret: <client secret> | pkceEnabled: <true/false> | pkceMethod: <PLAIN/S256/...>",
         "enabled":true
     }
 
@@ -105,6 +128,7 @@ The following options are available:
 .. list-table::
   :widths: 25 55 10 10
   :header-rows: 1
+  :align: left
 
   * - Option
     - Description
@@ -126,6 +150,14 @@ The following options are available:
     - The base URL of the OpenID Connect (OIDC) server as explained above.
     - Y
     - \-
+  * - ``dataverse.auth.oidc.pkce.enabled``
+    - Set to ``true`` to enable :ref:`PKCE <oidc-pkce>` in auth flow.
+    - N
+    - ``false``
+  * - ``dataverse.auth.oidc.pkce.method``
+    - Set code challenge method. Default equals best practice.
+    - N
+    - ``S256``
   * - ``dataverse.auth.oidc.title``
     - The UI visible name for this provider in login options.
     - N

From 5ed66e6bb481bd94c81965b4032e642a058943e4 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 15 May 2023 12:20:20 +0200
Subject: [PATCH 063/922] chore(deps): update to Testcontainers 1.18.1

---
 modules/dataverse-parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 7f8f73e049a..060fc22b4d2 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -167,7 +167,7 @@
         <gdcc.xoai.version>5.0.0</gdcc.xoai.version>
     
         <!-- Testing dependencies -->
-        <testcontainers.version>1.17.6</testcontainers.version>
+        <testcontainers.version>1.18.1</testcontainers.version>
         <smallrye-mpconfig.version>2.10.1</smallrye-mpconfig.version>
     
         <junit.version>4.13.1</junit.version>

From b9720c868b89e9db191b1425fb31574771bc1cee Mon Sep 17 00:00:00 2001
From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com>
Date: Mon, 15 May 2023 12:35:42 -0400
Subject: [PATCH 064/922] Update
 src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 .../java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index cd9311ec518..819a14c6c68 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -202,7 +202,6 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto)
        
         writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title), datasetDto.getMetadataLanguage());
         writeFullElement(xmlw, "subTitl", dto2Primitive(version, DatasetFieldConstant.subTitle));
-        //writeFullElement(xmlw, "altTitl", dto2Primitive(version, DatasetFieldConstant.alternativeTitle));
         FieldDTO altField = dto2FieldDTO( version, DatasetFieldConstant.alternativeTitle, "citation"  );
         if (altField != null) {
             writeMultipleElement(xmlw, "altTitl", altField, datasetDto.getMetadataLanguage());

From 5684140dff737f092f195ffeefddbf5074a409e5 Mon Sep 17 00:00:00 2001
From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com>
Date: Mon, 15 May 2023 12:37:18 -0400
Subject: [PATCH 065/922] Update
 src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 .../iq/dataverse/export/openaire/OpenAireExportUtil.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
index 334b18f4601..e858dee6d2b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
@@ -331,7 +331,7 @@ public static void writeTitlesElement(XMLStreamWriter xmlw, DatasetVersionDTO da
     private static boolean writeMultipleTitleElement(XMLStreamWriter xmlw, String titleType, DatasetVersionDTO datasetVersionDTO, String metadataBlockName, boolean title_check, String language) throws XMLStreamException {
         MetadataBlockDTO block = datasetVersionDTO.getMetadataBlocks().get(metadataBlockName);
         if (block != null) {
-            logger.info("Block is not empty");
+            logger.fine("Block is not empty");
             List<FieldDTO> fieldsBlock =  block.getFields();
             if (fieldsBlock != null) {
                 for (FieldDTO fieldDTO : fieldsBlock) {

From 64f4f1f0af83384b8157f2b13d29d941cb2aac77 Mon Sep 17 00:00:00 2001
From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com>
Date: Mon, 15 May 2023 12:37:31 -0400
Subject: [PATCH 066/922] Update
 src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 .../iq/dataverse/export/openaire/OpenAireExportUtil.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
index e858dee6d2b..146c442526a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
@@ -335,7 +335,7 @@ private static boolean writeMultipleTitleElement(XMLStreamWriter xmlw, String ti
             List<FieldDTO> fieldsBlock =  block.getFields();
             if (fieldsBlock != null) {
                 for (FieldDTO fieldDTO : fieldsBlock) {
-                    logger.info(titleType + " " + fieldDTO.getTypeName());
+                    logger.fine(titleType + " " + fieldDTO.getTypeName());
                     if (titleType.toLowerCase().equals(fieldDTO.getTypeName().toLowerCase())) {
                         logger.info("Found Alt title");
                         List<String> fields = fieldDTO.getMultiplePrimitive();

From 37a372a23c92d4eb2c946a50bf00b91319e141de Mon Sep 17 00:00:00 2001
From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com>
Date: Mon, 15 May 2023 12:37:44 -0400
Subject: [PATCH 067/922] Update
 src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 .../iq/dataverse/export/openaire/OpenAireExportUtil.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
index 146c442526a..037428d0ea1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
@@ -337,7 +337,7 @@ private static boolean writeMultipleTitleElement(XMLStreamWriter xmlw, String ti
                 for (FieldDTO fieldDTO : fieldsBlock) {
                     logger.fine(titleType + " " + fieldDTO.getTypeName());
                     if (titleType.toLowerCase().equals(fieldDTO.getTypeName().toLowerCase())) {
-                        logger.info("Found Alt title");
+                        logger.fine("Found Alt title");
                         List<String> fields = fieldDTO.getMultiplePrimitive();
                         for (String value : fields) {
                             if (!writeTitleElement(xmlw, titleType, value, title_check, language))

From 2ef0e5f0231367b90e962e73a638aab4c84a9ada Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Mon, 15 May 2023 12:39:32 -0400
Subject: [PATCH 068/922] test

---
 .../iq/dataverse/export/openaire/OpenAireExportUtil.java    | 2 --
 .../edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json | 6 ++++++
 .../edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml  | 2 ++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
index 334b18f4601..6dca1ac348a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/openaire/OpenAireExportUtil.java
@@ -322,8 +322,6 @@ public static void writeTitlesElement(XMLStreamWriter xmlw, DatasetVersionDTO da
         String subtitle = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.subTitle);
         title_check = writeTitleElement(xmlw, "Subtitle", subtitle, title_check, language);
 
-        //String alternativeTitle = dto2Primitive(datasetVersionDTO, DatasetFieldConstant.alternativeTitle);
-        //title_check = writeTitleElement(xmlw, "AlternativeTitle", alternativeTitle, title_check, language);
         title_check = writeMultipleTitleElement(xmlw, "AlternativeTitle", datasetVersionDTO, "citation", title_check, language);
         writeEndTag(xmlw, title_check);
     }
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json
index 7845f77d33f..9bdc7e45349 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json
+++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.json
@@ -29,6 +29,12 @@
             "typeClass": "primitive",
             "value": "Darwin's Finches"
           },
+          {
+              "typeName": "alternativeTitle",
+              "multiple": true,
+              "typeClass": "primitive",
+              "value": ["Darwin's Finches Alternative Title1", "Darwin's Finches Alternative Title2"]
+          },
           {
             "typeName": "author",
             "multiple": true,
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml
index 5bbfdae09ac..6730c44603a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml
+++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml
@@ -17,6 +17,8 @@
     <citation>
       <titlStmt>
         <titl xml:lang="en">Darwin's Finches</titl>
+        <altTitl>Darwin's Finches Alternative Title1</altTitl>
+        <altTitl>Darwin's Finches Alternative Title2</altTitl>
         <IDNo agency="DOI">doi:10.5072/FK2/PCA2E3</IDNo>
       </titlStmt>
       <rspStmt>

From 4b4c9155048f0ee074f6ba9d01a12e02ea4abd00 Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Mon, 15 May 2023 12:54:37 -0400
Subject: [PATCH 069/922] docs

---
 doc/release-notes/9428-alternative-title.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/9428-alternative-title.md b/doc/release-notes/9428-alternative-title.md
index d6eaa680612..3bc74f218b5 100644
--- a/doc/release-notes/9428-alternative-title.md
+++ b/doc/release-notes/9428-alternative-title.md
@@ -3,4 +3,7 @@ Alternative Title is made repeatable.
 `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
 - One will also need to update solr schema:
 Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml` 
-Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"` 
+Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
+
+Since Alternative Title is repeatable now, old json apis would not be compatable with a new version since value of alternative title has changed from simple string to an array.
+For example, instead "value": "Alternative Title", the value canbe "value": ["Alternative Title1", "Alternative Title2"]  

From 07c70d8933d550eb01f3bd2cfb3626b9a2d94f8c Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 13:01:01 -0500
Subject: [PATCH 070/922] flyway update script

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql   | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
new file mode 100644
index 00000000000..1ffc87dfa32
--- /dev/null
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -0,0 +1 @@
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);

From c8726958e45d6e25bc4ca773c55ce58afc1bf431 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 13:56:33 -0500
Subject: [PATCH 071/922] fix duplicate action

---
 src/main/webapp/filesFragment.xhtml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 6122b86b274..7395998042f 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -483,7 +483,8 @@
                                styleClass="btn btn-default btn-request"                                   
                                action="#{DatasetPage.requestAccessMultipleFiles()}"
                                update="@form, @([id$=messagePanel])"
-                               disabled="#{DatasetPage.locked}">
+                               disabled="#{DatasetPage.locked or !DatasetPage.fileAccessRequestMultiButtonEnabled}">
+                    <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle['file.requestAccess'])}"/>
                     <span class="glyphicon glyphicon-bullhorn"/> #{bundle['file.requestAccess']}
                 </p:commandLink>
                 <p:commandLink rendered="#{DatasetPage.fileAccessRequestMultiSignUpButtonRequired}"

From 6c0dab8c4771c96dce19f2e3a01cb2d175223730 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 13:57:13 -0500
Subject: [PATCH 072/922] add comment to check possible unused method

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 9294620d790..33b598083da 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3209,6 +3209,68 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
     }
 
 
+    /*helper function to filter the selected files into <selected downloadable>, 
+    and <selected, non downloadable> and <selected restricted> for reuse*/
+
+    private boolean filterSelectedFiles(){
+        setSelectedDownloadableFiles(new ArrayList<>());
+        setSelectedNonDownloadableFiles(new ArrayList<>());
+        setSelectedRestrictedFiles(new ArrayList<>());
+        setSelectedUnrestrictedFiles(new ArrayList<>());
+
+        boolean someFiles = false;
+        for (FileMetadata fmd : this.selectedFiles){
+            if(this.fileDownloadHelper.canDownloadFile(fmd)){
+                getSelectedDownloadableFiles().add(fmd);
+                someFiles=true;
+            } else {
+                getSelectedNonDownloadableFiles().add(fmd);
+            }
+            if(fmd.isRestricted()){
+                getSelectedRestrictedFiles().add(fmd); //might be downloadable to user or not
+                someFiles=true;
+            } else {
+                getSelectedUnrestrictedFiles().add(fmd);
+                someFiles=true;
+            }
+
+        }
+        return someFiles;
+    }
+//QDRADA - still needed?
+    public void validateFilesForRequestAccess(){
+        this.filterSelectedFiles();
+
+        if(!dataset.isFileAccessRequest()){ //is this needed? wouldn't be able to click Request Access if this !isFileAccessRequest()
+            return;
+        }
+
+        if(!this.selectedRestrictedFiles.isEmpty()){
+            ArrayList nonDownloadableRestrictedFiles = new ArrayList<>();
+
+            List<DataFile> userRequestedDataFiles = ((AuthenticatedUser) session.getUser()).getRequestedDataFiles();
+
+            for(FileMetadata fmd : this.selectedRestrictedFiles){
+                if(!this.fileDownloadHelper.canDownloadFile(fmd) && !userRequestedDataFiles.contains(fmd.getDataFile())){
+                    nonDownloadableRestrictedFiles.add(fmd);
+                }
+            }
+
+            if(!nonDownloadableRestrictedFiles.isEmpty()){
+                guestbookResponse.setDataFile(null);
+                guestbookResponse.setSelectedFileIds(this.getFilesIdsString(nonDownloadableRestrictedFiles));
+
+                if(this.isGuestbookAndTermsPopupRequired()){ //need to pop up the guestbook and terms dialog
+                    PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
+                } else {
+                    this.requestAccessMultipleFiles();
+                }
+            } else {
+                //popup select data files
+            }
+        }
+    }
+
     private boolean selectAllFiles;
 
     public boolean isSelectAllFiles() {

From 76b3b181d449336a9659bf5dc54637be90ff8679 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 13:57:30 -0500
Subject: [PATCH 073/922] debug logging

---
 .../iq/dataverse/engine/command/impl/RequestAccessCommand.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index b87b9a73aa5..df7c7367f2d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -52,6 +52,8 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         }
         
         //if user already has permission to download file or the file is public throw command exception
+        logger.info("User: " + this.getRequest().getAuthenticatedUser().getName());
+        logger.info("File: " + file.getId());
         if (!file.isRestricted() || ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)) {
             throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.alreadyHasDownloadPermisssion"), this);
         }

From bd603ec7a803542460eea3a66600572bdf85f57a Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 14:17:45 -0500
Subject: [PATCH 074/922] more debug

---
 .../command/impl/RequestAccessCommand.java       | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index df7c7367f2d..d710ed66551 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -44,6 +44,19 @@ public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean s
         this.sendNotification = sendNotification;
     }
 
+        
+        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) {
+            this(dvRequest, file, gbr, false);
+        }
+
+        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, Boolean sendNotification) {
+            // for data file check permission on owning dataset
+            super(dvRequest, file);        
+            this.file = file;
+            this.requester = (AuthenticatedUser) dvRequest.getUser();
+            this.fileAccessRequest = new FileAccessRequest(file,requester,gbr);
+            this.sendNotification = sendNotification;
+        }
     @Override
     public DataFile execute(CommandContext ctxt) throws CommandException {
 
@@ -53,7 +66,8 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         
         //if user already has permission to download file or the file is public throw command exception
         logger.info("User: " + this.getRequest().getAuthenticatedUser().getName());
-        logger.info("File: " + file.getId());
+        logger.info("File: " + file.getId() + " : restricted?: " + file.isRestricted());
+        logger.info("permission?: " + ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile));
         if (!file.isRestricted() || ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)) {
             throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.alreadyHasDownloadPermisssion"), this);
         }

From 5e29a0600b6d4fe59d89191897bc61cd45d78494 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 10:20:39 -0500
Subject: [PATCH 075/922] more debug

---
 .../edu/harvard/iq/dataverse/DataFile.java    |  11 +
 .../iq/dataverse/DataFileServiceBean.java     |  53 ++++
 .../edu/harvard/iq/dataverse/Dataset.java     |   6 +
 .../edu/harvard/iq/dataverse/DatasetPage.java | 134 +++++++--
 .../iq/dataverse/FileAccessRequest.java       | 262 ++++++++++++++----
 .../FileAccessRequestServiceBean.java         |  89 ++++++
 .../iq/dataverse/FileDownloadHelper.java      |  30 +-
 .../iq/dataverse/FileDownloadServiceBean.java | 102 ++++++-
 .../edu/harvard/iq/dataverse/FilePage.java    |   6 +
 .../iq/dataverse/GuestbookResponse.java       |  13 +-
 .../iq/dataverse/ManagePermissionsPage.java   |  47 ++++
 .../iq/dataverse/UserNotification.java        |   2 +-
 .../UserNotificationServiceBean.java          |  22 ++
 .../users/AuthenticatedUser.java              |  25 ++
 .../command/impl/RequestAccessCommand.java    |  33 ++-
 .../harvard/iq/dataverse/util/FileUtil.java   |  66 +++++
 .../harvard/iq/dataverse/util/MailUtil.java   |   2 +
 src/main/webapp/dataset.xhtml                 |  24 +-
 .../file-download-button-fragment.xhtml       |  74 ++---
 src/main/webapp/file.xhtml                    |  40 ++-
 src/main/webapp/filesFragment.xhtml           |   9 +-
 ...l => guestbook-terms-popup-fragment.xhtml} |  17 ++
 .../iq/dataverse/util/FileUtilTest.java       |   2 +-
 23 files changed, 896 insertions(+), 173 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java
 rename src/main/webapp/{file-request-access-popup-fragment.xhtml => guestbook-terms-popup-fragment.xhtml} (65%)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index 28d814d9844..c43800c57ce 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -200,6 +200,17 @@ public String toString() {
     @OneToMany(mappedBy="dataFile", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
     private List<GuestbookResponse> guestbookResponses;
 
+    @OneToMany(mappedBy="dataFile",fetch = FetchType.LAZY,cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REFRESH})
+    private List<FileAccessRequest> fileAccessRequests;
+
+    public List<FileAccessRequest> getFileAccessRequests(){
+        return fileAccessRequests;
+    }
+
+    public void setFileAccessRequests(List<FileAccessRequest> fARs){
+        this.fileAccessRequests = fARs;
+    }
+    
     public List<GuestbookResponse> getGuestbookResponses() {
         return guestbookResponses;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 196f84b6877..449e8d351c6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -154,6 +154,27 @@ public DataFile find(Object pk) {
         
     }*/
     
+    public List<DataFile> findAll(List<Long> fileIds){
+        List<DataFile> dataFiles = new ArrayList<>();
+
+         for (Long fileId : fileIds){
+             dataFiles.add(find(fileId));
+         }
+
+        return dataFiles;
+    }
+
+    public List<DataFile> findAll(String fileIdsAsString){
+        ArrayList<Long> dataFileIds = new ArrayList<>();
+
+        String[] fileIds = fileIdsAsString.split(",");
+        for (String fId : fileIds){
+            dataFileIds.add(Long.parseLong(fId));
+        }
+
+        return findAll(dataFileIds);
+    }
+    
     public DataFile findByGlobalId(String globalId) {
             return (DataFile) dvObjectService.findByGlobalId(globalId, DvObject.DType.DataFile);
     }
@@ -350,6 +371,18 @@ public FileMetadata findMostRecentVersionFileIsIn(DataFile file) {
             return fileMetadatas.get(0);
         }
     }
+    
+    public List<DataFile> findAllCheapAndEasy(String fileIdsAsString){ 
+        //assumption is that the fileIds are separated by ','
+        ArrayList <DataFile> dataFilesFound = new ArrayList<>();
+        String[] fileIds = fileIdsAsString.split(",");
+        DataFile df = this.findCheapAndEasy(Long.parseLong(fileIds[0]));
+        if(df != null){
+            dataFilesFound.add(df);
+        }
+
+        return dataFilesFound;
+    }
 
     public DataFile findCheapAndEasy(Long id) {
         DataFile dataFile;
@@ -802,6 +835,7 @@ public void findFileMetadataOptimizedExperimental(Dataset owner, DatasetVersion
                 dataFile.addFileAccessRequester(au);
             }
 
+            dataFile.setFileAccessRequesters(retrieveFileAccessRequesters(dataFile));
             dataFiles.add(dataFile);
             filesMap.put(dataFile.getId(), i++);
         }
@@ -821,6 +855,25 @@ public void findFileMetadataOptimizedExperimental(Dataset owner, DatasetVersion
         owner.setFiles(dataFiles);
     }
     
+    private List<AuthenticatedUser> retrieveFileAccessRequesters(DataFile fileIn) {
+        List<AuthenticatedUser> retList = new ArrayList<>();
+
+        // List<Object> requesters = em.createNativeQuery("select authenticated_user_id
+        // from fileaccessrequests where datafile_id =
+        // "+fileIn.getId()).getResultList();
+        List<Object> requesters = em.createNativeQuery("select authenticated_user_id from fileaccessrequests where datafile_id = " + fileIn.getId() + " and request_state='CREATED'").getResultList();
+
+        for (Object userIdObj : requesters) {
+            Long userId = (Long) userIdObj;
+            AuthenticatedUser user = userService.find(userId);
+            if (user != null) {
+                retList.add(user);
+            }
+        }
+
+        return retList;
+    }
+    
     private List<FileMetadata> retrieveFileMetadataForVersion(Dataset dataset, DatasetVersion version, List<DataFile> dataFiles, Map<Long, Integer> filesMap, Map<Long, Integer> categoryMap) {
         List<FileMetadata> retList = new ArrayList<>();
         Map<Long, Set<Long>> categoryMetaMap = new HashMap<>();
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 683b6687c8b..305e9a404e7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -858,6 +858,12 @@ public String getHarvestingDescription() {
         return null;
     }
 
+    public boolean hasEnabledGuestbook(){
+        Guestbook gb = this.getGuestbook();
+
+        return ( gb != null && gb.isEnabled());
+    }
+    
     @Override
     public boolean equals(Object object) {
         // TODO: Warning - this method won't work in the case the id fields are not set
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 33b598083da..8754c26a3b6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -366,6 +366,19 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
         this.showIngestSuccess = showIngestSuccess;
     }
 
+    private String termsGuestbookPopupAction = "";
+
+    public void setTermsGuestbookPopupAction(String popupAction){
+        if(popupAction != null && popupAction.length() > 0){
+            this.termsGuestbookPopupAction = popupAction;
+        }
+
+    }
+
+    public String getTermsGuestbookPopupAction(){
+        return termsGuestbookPopupAction;
+    }
+    
     // TODO: Consider renaming "configureTools" to "fileConfigureTools".
     List<ExternalTool> configureTools = new ArrayList<>();
     // TODO: Consider renaming "exploreTools" to "fileExploreTools".
@@ -3137,9 +3150,14 @@ public void setValidateFilesOutcome(String validateFilesOutcome) {
         this.validateFilesOutcome = validateFilesOutcome;
     }
 
-    public boolean validateFilesForDownload(boolean guestbookRequired, boolean downloadOriginal) {
-        setSelectedDownloadableFiles(new ArrayList<>());
-        setSelectedNonDownloadableFiles(new ArrayList<>());
+    public boolean validateFilesForDownload(boolean downloadOriginal){ 
+        if (this.selectedFiles.isEmpty()) {
+            PrimeFaces.current().executeScript("PF('selectFilesForDownload').show()");
+            return false;
+        } else {
+            this.filterSelectedFiles();
+        }
+        
         //assume Pass unless something bad happens
         setValidateFilesOutcome("Pass");
         Long bytes = (long) 0;
@@ -3170,6 +3188,13 @@ public boolean validateFilesForDownload(boolean guestbookRequired, boolean downl
             return false;
         }
 
+        
+//QDRADA handle new state from
+        /*if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()){
+            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
+        }
+        */
+        
         // If some of the files were restricted and we had to drop them off the
         // list, and NONE of the files are left on the downloadable list
         // - we show them a "you're out of luck" popup:
@@ -3183,10 +3208,11 @@ public boolean validateFilesForDownload(boolean guestbookRequired, boolean downl
             return true;
         }
 
-        if (guestbookRequired) {
+      //QDRADA - still needed?
+/*        if (guestbookRequired) {
             setValidateFilesOutcome("GuestbookRequired");
         }
-
+*/
         return true;
 
     }
@@ -3208,6 +3234,67 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
         guestbookResponse.setDownloadtype("Download");
     }
 
+    /*helper function to filter the selected files into <selected downloadable>, 
+    and <selected, non downloadable> and <selected restricted> for reuse*/
+
+    private boolean filterSelectedFiles(){
+        setSelectedDownloadableFiles(new ArrayList<>());
+        setSelectedNonDownloadableFiles(new ArrayList<>());
+        setSelectedRestrictedFiles(new ArrayList<>());
+        setSelectedUnrestrictedFiles(new ArrayList<>());
+
+        boolean someFiles = false;
+        for (FileMetadata fmd : this.selectedFiles){
+            if(this.fileDownloadHelper.canDownloadFile(fmd)){
+                getSelectedDownloadableFiles().add(fmd);
+                someFiles=true;
+            } else {
+                getSelectedNonDownloadableFiles().add(fmd);
+            }
+            if(fmd.isRestricted()){
+                getSelectedRestrictedFiles().add(fmd); //might be downloadable to user or not
+                someFiles=true;
+            } else {
+                getSelectedUnrestrictedFiles().add(fmd);
+                someFiles=true;
+            }
+
+        }
+        return someFiles;
+    }
+
+    public void validateFilesForRequestAccess(){
+        this.filterSelectedFiles();
+
+        if(!dataset.isFileAccessRequest()){ //is this needed? wouldn't be able to click Request Access if this !isFileAccessRequest()
+            return;
+        }
+
+        if(!this.selectedRestrictedFiles.isEmpty()){
+            ArrayList nonDownloadableRestrictedFiles = new ArrayList<>();
+
+            List<DataFile> userRequestedDataFiles = ((AuthenticatedUser) session.getUser()).getRequestedDataFiles();
+
+            for(FileMetadata fmd : this.selectedRestrictedFiles){
+                if(!this.fileDownloadHelper.canDownloadFile(fmd) && !userRequestedDataFiles.contains(fmd.getDataFile())){
+                    nonDownloadableRestrictedFiles.add(fmd);
+                }
+            }
+
+            if(!nonDownloadableRestrictedFiles.isEmpty()){
+                guestbookResponse.setDataFile(null);
+                guestbookResponse.setSelectedFileIds(this.getFilesIdsString(nonDownloadableRestrictedFiles));
+
+                if(this.isGuestbookAndTermsPopupRequired()){ //need to pop up the guestbook and terms dialog
+                    PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
+                } else {
+                    this.requestAccessMultipleFiles();
+                }
+            } else {
+                //popup select data files
+            }
+        }
+    }
 
     /*helper function to filter the selected files into <selected downloadable>, 
     and <selected, non downloadable> and <selected restricted> for reuse*/
@@ -3295,26 +3382,23 @@ public void toggleAllSelected(){
 
     // helper Method
     public String getSelectedFilesIdsString() {
-        String downloadIdString = "";
-        for (FileMetadata fmd : this.selectedFiles){
-            if (!StringUtil.isEmpty(downloadIdString)) {
-                downloadIdString += ",";
-            }
-            downloadIdString += fmd.getDataFile().getId();
-        }
-        return downloadIdString;
+        return this.getFilesIdsString(this.selectedFiles);
     }
-
+    
     // helper Method
     public String getSelectedDownloadableFilesIdsString() {
-        String downloadIdString = "";
-        for (FileMetadata fmd : this.selectedDownloadableFiles){
-            if (!StringUtil.isEmpty(downloadIdString)) {
-                downloadIdString += ",";
+        return this.getFilesIdsString(this.selectedDownloadableFiles);
+    }
+    
+    public String getFilesIdsString(List<FileMetadata> fileMetadatas){ //for reuse
+        String idString = "";
+        for (FileMetadata fmd : fileMetadatas){
+            if (!StringUtil.isEmpty(idString)) {
+                idString += ",";
             }
-            downloadIdString += fmd.getDataFile().getId();
+            idString += fmd.getDataFile().getId();
         }
-        return downloadIdString;
+        return idString;
     }
 
 
@@ -5221,6 +5305,10 @@ public boolean isDownloadPopupRequired() {
     public boolean isRequestAccessPopupRequired() {
         return FileUtil.isRequestAccessPopupRequired(workingVersion);
     }
+    
+    public boolean isGuestbookAndTermsPopupRequired() {  
+        return FileUtil.isGuestbookAndTermsPopupRequired(workingVersion);
+    }
 
     public String requestAccessMultipleFiles() {
 
@@ -5236,11 +5324,11 @@ public String requestAccessMultipleFiles() {
             for (FileMetadata fmd : selectedFiles){
                  fileDownloadHelper.addMultipleFilesForRequestAccess(fmd.getDataFile());
             }
-            if (isRequestAccessPopupRequired()) {
+            if (isGuestbookAndTermsPopupRequired()) {
                 //RequestContext requestContext = RequestContext.getCurrentInstance();
-                PrimeFaces.current().executeScript("PF('requestAccessPopup').show()");
+                PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show()"); //the popup will call writeGuestbookAndRequestAccess();
                 return "";
-            } else {
+            }else {
                 //No popup required
                 fileDownloadHelper.requestAccessIndirect();
                 return "";
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
index 76c5df4409a..723a54c8587 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
@@ -1,91 +1,237 @@
 package edu.harvard.iq.dataverse;
 
-import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-
-import javax.persistence.Column;
-import javax.persistence.Embeddable;
-import javax.persistence.EmbeddedId;
+import java.io.Serializable;
+import javax.persistence.CascadeType;
+import javax.persistence.GeneratedValue;
+import javax.persistence.UniqueConstraint;
 import javax.persistence.Entity;
+import javax.persistence.Table;
+import javax.persistence.Index;
+import javax.persistence.Id;
 import javax.persistence.JoinColumn;
+import javax.persistence.JoinTable;
 import javax.persistence.ManyToOne;
-import javax.persistence.MapsId;
-import javax.persistence.Table;
-import javax.persistence.Temporal;
-import javax.persistence.TemporalType;
-import java.io.Serializable;
-import java.util.Date;
+import javax.persistence.OneToOne;
+import javax.persistence.EnumType;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.datavariable.DataVariable;
+import javax.persistence.Column;
+import javax.persistence.Enumerated;
+import javax.persistence.GenerationType;
+import javax.persistence.NamedQueries;
+import javax.persistence.NamedQuery;
+
+/**
+ *
+ * @author Marina
+ */
 
 @Entity
-@Table(name = "fileaccessrequests")
-public class FileAccessRequest {
-    @EmbeddedId
-    private FileAccessRequestKey id;
+@Table(name = "fileaccessrequests", //having added the guestbookresponse_id column to fileaccessrequests
+    uniqueConstraints=@UniqueConstraint(columnNames={"datafile_id", "authenticated_user_id","request_state"}) //this may not make sense at some future point
+)
+
+@NamedQueries({
+        @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserId",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId"),
+        @NamedQuery(name = "FileAccessRequest.findByGuestbookResponseId",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.guestbookResponse.id=:guestbookResponseId"),
+        @NamedQuery(name = "FileAccessRequest.findByDataFileId",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.dataFile.id=:dataFileId"),
+        @NamedQuery(name = "FileAccessRequest.findByRequestState",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.requestState=:requestState"),
+        @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserIdAndRequestState",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId and far.requestState=:requestState"),
+        @NamedQuery(name = "FileAccessRequest.findByGuestbookResponseIdAndRequestState",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.guestbookResponse.id=:guestbookResponseId and far.requestState=:requestState"),
+        @NamedQuery(name = "FileAccessRequest.findByDataFileIdAndRequestState",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.dataFile.id=:dataFileId and far.requestState=:requestState"),
+        @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserIdAndDataFileIdAndRequestState",
+                query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId and far.dataFile.id=:dataFileId and far.requestState=:requestState")
+})
+
+public class FileAccessRequest implements Serializable{
+    private static final long serialVersionUID = 1L;
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
     @ManyToOne
-    @MapsId("dataFile")
-    @JoinColumn(name = "datafile_id")
+    @JoinColumn(nullable=false)
     private DataFile dataFile;
+    
     @ManyToOne
-    @MapsId("authenticatedUser")
-    @JoinColumn(name = "authenticated_user_id")
-    private AuthenticatedUser authenticatedUser;
-
-    @Temporal(value = TemporalType.TIMESTAMP)
-    @Column(name = "creation_time")
-    private Date creationTime;
-
-    public FileAccessRequestKey getId() {
+    @JoinColumn(name="authenticated_user_id",nullable=false)
+    private AuthenticatedUser user;
+    
+    @OneToOne
+    @JoinColumn(nullable=true)
+    private GuestbookResponse guestbookResponse;
+    
+    public enum RequestState {CREATED,EDITED,GRANTED,REJECTED,RESUBMIT,INVALIDATED,CLOSED};
+    //private RequestState state;
+    @Enumerated(EnumType.STRING)
+    @Column(name="request_state", nullable=false )
+    private RequestState requestState;
+    
+    public FileAccessRequest(){
+        
+    }
+    
+    public FileAccessRequest(DataFile df, AuthenticatedUser au){
+        setDataFile(df);
+        setRequester(au);
+        setState(RequestState.CREATED);
+    }
+    
+    public FileAccessRequest(DataFile df, AuthenticatedUser au, GuestbookResponse gbr){
+        setDataFile(df);
+        setRequester(au);
+        setGuestbookResponse(gbr);
+        setState(RequestState.CREATED);
+    }
+     
+    public Long getId() {
         return id;
     }
 
-    public void setId(FileAccessRequestKey id) {
+    public void setId(Long id) {
         this.id = id;
     }
-
-    public DataFile getDataFile() {
+    
+    public DataFile getDataFile(){
         return dataFile;
     }
-
-    public void setDataFile(DataFile dataFile) {
-        this.dataFile = dataFile;
+    
+    public final void setDataFile(DataFile df){
+        this.dataFile = df;
+    }
+    
+    public AuthenticatedUser getRequester(){
+        return user;
+    }
+    
+    public final void setRequester(AuthenticatedUser au){
+        this.user = au;
+    }
+    
+    public GuestbookResponse getGuestbookResponse(){
+        return guestbookResponse;
+    }
+    
+    public final void setGuestbookResponse(GuestbookResponse gbr){
+        this.guestbookResponse = gbr;
+    }
+    
+    public RequestState getState() {
+        return this.requestState;
+    }
+    
+    public void setState(RequestState requestState) {
+        this.requestState = requestState;
+    }
+    
+    public String getStateLabel() {
+        if(isStateCreated()){
+            return "created";
+        }
+        if(isStateEdited()) {
+            return "edited";
+        }
+        if(isStateGranted()) {
+            return "granted";
+        }
+        if(isStateRejected()) {
+            return "rejected";
+        }
+        if(isStateResubmit()) {
+            return "resubmit";
+        }
+        if(isStateInvalidated()) {
+            return "invalidated";
+        }
+        if(isStateClosed()) {
+            return "closed";
+        }
+        return null; 
+    }
+    
+    public void setStateCreated() {
+        this.requestState = RequestState.CREATED;
+    }
+    
+    public void setStateEdited() {
+        this.requestState = RequestState.EDITED;
+    }
+    
+    public void setStateGranted() {
+        this.requestState = RequestState.GRANTED;
     }
 
-    public AuthenticatedUser getAuthenticatedUser() {
-        return authenticatedUser;
+    public void setStateRejected() {
+        this.requestState = RequestState.REJECTED;
     }
 
-    public void setAuthenticatedUser(AuthenticatedUser authenticatedUser) {
-        this.authenticatedUser = authenticatedUser;
+    public void setStateResubmit() {
+        this.requestState = RequestState.RESUBMIT;
+    }
+    
+    public void setStateInvalidated() {
+        this.requestState = RequestState.INVALIDATED;
     }
 
-    public Date getCreationTime() {
-        return creationTime;
+    public void setStateClosed() {
+        this.requestState = RequestState.CLOSED;
     }
 
-    public void setCreationTime(Date creationTime) {
-        this.creationTime = creationTime;
+    
+    public boolean isStateCreated() {
+        return this.requestState == RequestState.CREATED;
+    }
+   
+    public boolean isStateEdited() {
+        return this.requestState == RequestState.EDITED;
+    }
+    
+    public boolean isStateGranted() {
+        return this.requestState == RequestState.GRANTED;
     }
 
-    @Embeddable
-    public static class FileAccessRequestKey implements Serializable {
-        @Column(name = "datafile_id")
-        private Long dataFile;
-        @Column(name = "authenticated_user_id")
-        private Long authenticatedUser;
+    public boolean isStateRejected() {
+        return this.requestState == RequestState.REJECTED;
+    }
 
-        public Long getDataFile() {
-            return dataFile;
-        }
+    public boolean isStateResubmit() {
+        return this.requestState == RequestState.RESUBMIT;
+    }
+    
+    public boolean isStateInvalidated() {
+        return this.requestState == RequestState.INVALIDATED;
+    }
 
-        public void setDataFile(Long dataFile) {
-            this.dataFile = dataFile;
-        }
+    public boolean isStateClosed() {
+        return this.requestState == RequestState.CLOSED;
+    }
+    
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += (id != null ? id.hashCode() : 0);
+        return hash;
+    }
 
-        public Long getAuthenticatedUser() {
-            return authenticatedUser;
+    @Override
+    public boolean equals(Object object) {
+        // TODO: Warning - this method won't work in the case the id fields are not set
+        if (!(object instanceof FileAccessRequest)) {
+            return false;
         }
-
-        public void setAuthenticatedUser(Long authenticatedUser) {
-            this.authenticatedUser = authenticatedUser;
+        FileAccessRequest other = (FileAccessRequest) object;
+        if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
+            return false;
         }
+        return true;
     }
-}
+    
+    
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java
new file mode 100644
index 00000000000..215e4695a75
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java
@@ -0,0 +1,89 @@
+package edu.harvard.iq.dataverse;
+
+import java.util.List;
+import javax.ejb.Stateless;
+import javax.inject.Named;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+
+/**
+ *
+ * @author Marina
+ */
+@Stateless
+@Named
+public class FileAccessRequestServiceBean {
+
+    @PersistenceContext(unitName = "VDCNet-ejbPU")
+    private EntityManager em;
+    
+    public FileAccessRequest find(Object pk) {
+        return em.find(FileAccessRequest.class, pk);
+    }
+
+    public List<FileAccessRequest> findAll() {
+        return em.createQuery("select object(o) from FileAccessRequest as o order by o.id", FileAccessRequest.class).getResultList();
+    }
+    
+    public List<FileAccessRequest> findAll(Long authenticatedUserId, Long fileId, FileAccessRequest.RequestState requestState){
+        return em.createNamedQuery("FileAccessRequest.findByAuthenticatedUserIdAndDataFileIdAndRequestState", FileAccessRequest.class)
+                .setParameter("authenticatedUserId",authenticatedUserId)
+                .setParameter("dataFileId",fileId)
+                .setParameter("requestState",requestState)
+                .getResultList();
+    }
+    
+    public List<FileAccessRequest> findAllByAuthenticedUserId(Long authenticatedUserId){
+        return em.createNamedQuery("FileAccessRequest.findByAuthenticatedUserId", FileAccessRequest.class)
+                        .setParameter("authenticatedUserId", authenticatedUserId)
+                        .getResultList();
+    }
+    
+    public List<FileAccessRequest> findAllByGuestbookResponseId(Long guestbookResponseId){
+        return em.createNamedQuery("FileAccessRequest.findByGuestbookResponseId", FileAccessRequest.class)
+                        .setParameter("guestbookResponseId", guestbookResponseId)
+                        .getResultList();
+    
+    }
+    
+    public List<FileAccessRequest> findAllByDataFileId(Long dataFileId){
+        return em.createNamedQuery("FileAccessRequest.findByDataFileId", FileAccessRequest.class)
+                        .setParameter("dataFileId", dataFileId)
+                        .getResultList();
+    }
+    
+    public List<FileAccessRequest> findAllByAuthenticatedUserIdAndRequestState(Long authenticatedUserId, FileAccessRequest.RequestState requestState){
+        return em.createNamedQuery("FileAccessRequest.findByAuthenticatedUserIdAndRequestState", FileAccessRequest.class)
+                        .setParameter("authenticatedUserId", authenticatedUserId)
+                        .setParameter("requestState",requestState)
+                        .getResultList();
+    }
+    
+    public List<FileAccessRequest> findAllByGuestbookResponseIdAndRequestState(Long guestbookResponseId, FileAccessRequest.RequestState requestState){
+        return em.createNamedQuery("FileAccessRequest.findByGuestbookResponseIdAndRequestState", FileAccessRequest.class)
+                        .setParameter("dataFileId", guestbookResponseId)
+                        .setParameter("requestState",requestState)
+                        .getResultList();
+    }
+    
+    public List<FileAccessRequest> findAllByDataFileIdAndRequestState(Long dataFileId, FileAccessRequest.RequestState requestState){
+        return em.createNamedQuery("FileAccessRequest.findByDataFileIdAndRequestState", FileAccessRequest.class)
+                        .setParameter("dataFileId", dataFileId)
+                        .setParameter("requestState",requestState)
+                        .getResultList();
+    }
+    
+    
+    public FileAccessRequest save(FileAccessRequest far) {
+        if (far.getId() == null) {
+            em.persist(far);
+            return far;
+        } else {
+            return em.merge(far);
+        }
+    }
+  
+    
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index 850efc2f1ae..e44aeafcc4d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -71,7 +71,7 @@ private boolean testResponseLength(String value) {
     // file downloads and multiple (batch) downloads - sice both use the same
     // terms/etc. popup.
     public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) {
-        PrimeFaces.current().executeScript("PF('downloadPopup').hide()");
+        PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
         guestbookResponse.setDownloadtype("Download");
          // Note that this method is only ever called from the file-download-popup -
          // meaning we know for the fact that we DO want to save this
@@ -91,7 +91,7 @@ public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse)
 
      public void writeGuestbookAndOpenSubset(GuestbookResponse guestbookResponse) {
 
-             PrimeFaces.current().executeScript("PF('downloadPopup').hide()");
+             PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
              PrimeFaces.current().executeScript("PF('downloadDataSubsetPopup').show()");
              guestbookResponse.setDownloadtype("Subset");
              fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
@@ -132,17 +132,23 @@ public void writeGuestbookAndLaunchExploreTool(GuestbookResponse guestbookRespon
 
          fileDownloadService.explore(guestbookResponse, fmd, externalTool);
          //requestContext.execute("PF('downloadPopup').hide()");
-         PrimeFaces.current().executeScript("PF('downloadPopup').hide()");
+         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
     }
      
     public void writeGuestbookAndLaunchPackagePopup(GuestbookResponse guestbookResponse) {
 
-            PrimeFaces.current().executeScript("PF('downloadPopup').hide()");
+            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
             PrimeFaces.current().executeScript("PF('downloadPackagePopup').show()");
             PrimeFaces.current().executeScript("handleResizeDialog('downloadPackagePopup')");
             fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
     }
 
+    public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookResponse) {
+            //requestContext.execute("PF('guestbookAndTermsPopup').hide()");
+            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
+            fileDownloadService.writeGuestbookResponseAndRequestAccess(guestbookResponse);
+    }
+    
      /**
       * Writes a guestbook entry for either popup scenario: guestbook or terms.
       */
@@ -307,13 +313,13 @@ public void requestAccessMultiple(List<DataFile> files) {
              }
          }
          if (notificationFile != null && succeeded) {
-             fileDownloadService.sendRequestFileAccessNotification(notificationFile, (AuthenticatedUser) session.getUser());
+             fileDownloadService.sendRequestFileAccessNotification(notificationFile.getOwner(), notificationFile.getId(), (AuthenticatedUser) session.getUser());
          }
      }
     
      public void requestAccessIndirect() {
          //Called when there are multiple files and no popup
-         // or there's a popup with sigular or multiple files
+         // or there's a popup with singular or multiple files
          // The list of files for Request Access is set in the Dataset Page when
          // user clicks the request access button in the files fragment
          // (and has selected one or more files)
@@ -329,8 +335,16 @@ private boolean processRequestAccess(DataFile file, Boolean sendNotification) {
 
              // create notification if necessary
              if (sendNotification) {
-                 fileDownloadService.sendRequestFileAccessNotification(file, user);
-             }           
+                 fileDownloadService.sendRequestFileAccessNotification(file.getOwner(), file.getId(), (AuthenticatedUser) session.getUser());
+             }
+             //ToDO QDRADA - where to write the response?
+             /*
+             //write the guestbookResponse if there is an enabled guestbook
+             GuestbookResponse gbr = this.getGuestbookResponse(); //can we be sure this is the correct guestbookResponse?? - can it get out of sync??
+             if( gbr != null && gbr.getGuestbook().isEnabled() ){
+                 fileDownloadService.writeGuestbookResponseRecordForRequestAccess(gbr); 
+             }
+             */
              JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success"));
              return true;
          }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index a90489be29a..f7612300eaf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -76,6 +76,8 @@ public class FileDownloadServiceBean implements java.io.Serializable {
     PrivateUrlServiceBean privateUrlService;
     @EJB
     SettingsServiceBean settingsService;
+    @EJB
+    MailServiceBean mailService;
 
     @Inject
     DataverseSession session;
@@ -192,6 +194,38 @@ public void writeGuestbookAndStartFileDownload(GuestbookResponse guestbookRespon
         redirectToDownloadAPI(guestbookResponse.getFileFormat(), guestbookResponse.getDataFile().getId());
         logger.fine("issued file download redirect for datafile "+guestbookResponse.getDataFile().getId());
     }
+    
+    public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookResponse){
+        if (guestbookResponse == null || ( guestbookResponse.getDataFile() == null && guestbookResponse.getSelectedFileIds() == null) ) {
+            return;
+        }
+
+        List <DataFile> selectedDataFiles = new ArrayList<>(); //always make sure it's at least an empty List
+
+        if(guestbookResponse.getDataFile() != null ){ //one file 'selected' by 'Request Access' button click
+            selectedDataFiles.add(datafileService.find(guestbookResponse.getDataFile().getId())); //don't want the findCheapAndEasy
+        }
+
+        if(guestbookResponse.getSelectedFileIds() != null && !guestbookResponse.getSelectedFileIds().isEmpty()) { //multiple selected through multi-select REquest Access button   
+            selectedDataFiles = datafileService.findAll(guestbookResponse.getSelectedFileIds());
+        }
+
+        int countRequestAccessSuccess = 0;
+
+        for(DataFile dataFile : selectedDataFiles){
+            guestbookResponse.setDataFile(dataFile);
+            writeGuestbookResponseRecordForRequestAccess(guestbookResponse);
+            if(requestAccess(dataFile,guestbookResponse)){
+                countRequestAccessSuccess++;
+            }
+        }
+
+        if(countRequestAccessSuccess > 0){
+            DataFile firstDataFile = selectedDataFiles.get(0);
+            sendRequestFileAccessNotification(firstDataFile.getOwner(), firstDataFile.getId(), (AuthenticatedUser) session.getUser());
+        }
+
+    }
 
     public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format) {
         if(!fileMetadata.getDatasetVersion().isDraft()){           
@@ -221,6 +255,18 @@ public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse) {
         }
     }
     
+    public void writeGuestbookResponseRecordForRequestAccess(GuestbookResponse guestbookResponse) {
+        try {
+            CreateGuestbookResponseCommand cmd = new CreateGuestbookResponseCommand(dvRequestService.getDataverseRequest(), guestbookResponse, guestbookResponse.getDataset());
+            commandEngine.submit(cmd);
+
+        } catch (CommandException e) {
+            //if an error occurs here then download won't happen no need for response recs...
+            logger.info("Failed to writeGuestbookResponseRecord for RequestAccess");
+        }
+
+    }
+    
     // The "guestBookRecord(s)AlreadyWritten" parameter in the 2 methods 
     // below (redirectToBatchDownloadAPI() and redirectToDownloadAPI(), for the 
     // multiple- and single-file downloads respectively) are passed to the 
@@ -499,15 +545,63 @@ public boolean requestAccess(Long fileId) {
             }             
         }        
         return false;
-    }    
+    }
+    
+    public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){
+        boolean accessRequested = false;
+        if (dvRequestService.getDataverseRequest().getAuthenticatedUser() == null){
+            return accessRequested;
+        }
+
+        List<AuthenticatedUser> fARs = dataFile.getFileAccessRequesters();
+
+        if(fARs.isEmpty() || (!fARs.isEmpty() && !fARs.contains((AuthenticatedUser)session.getUser()))){
+            try {
+                commandEngine.submit(new RequestAccessCommand(dvRequestService.getDataverseRequest(), dataFile, gbr));
+                accessRequested = true;
+            } catch (CommandException ex) {
+                logger.info("Unable to request access for file id " + dataFile.getId() + ". Exception: " + ex);
+            }
+        } 
+
+        return accessRequested;
+    }
     
-    public void sendRequestFileAccessNotification(DataFile datafile, AuthenticatedUser requestor) {
-        permissionService.getUsersWithPermissionOn(Permission.ManageFilePermissions, datafile).stream().forEach((au) -> {
-            userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REQUESTFILEACCESS, datafile.getId(), null, requestor, false);
+    public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) {
+        permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> {
+            userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, false);
         });
 
     } 
     
+    public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, GuestbookResponse gb){
+        Timestamp ts = new Timestamp(new Date().getTime()); 
+        UserNotification un = null;
+
+        //String appendMsgText = (gb == null)?("") : this.getGuestbookAppendEmailDetails(gb);
+        String appendMsgText = "";
+
+        //first send a notification for all the Users that have ManageDatasetPermissions a notification that a user has requested accedd    
+        List<AuthenticatedUser> mngDsPermUsers = permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset);
+
+        for (AuthenticatedUser au : mngDsPermUsers){
+            un = userNotificationService.sendUserNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId);
+
+            if(un != null){
+
+               boolean mailed = mailService.sendNotificationEmail(un, appendMsgText, (AuthenticatedUser)session.getUser(),false);
+               if(mailed){
+                   un.setEmailed(true);
+                   userNotificationService.save(un);
+               }    
+            }
+        }
+
+        //send the user that requested access a notification that they requested the access
+        userNotificationService.sendNotification((AuthenticatedUser) session.getUser(), ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId);
+    }
+ 
+    
     public String generateServiceKey() {
         UUID uid = UUID.randomUUID();
         // last 8 bytes, of the random UUID, 16 hex digits: 
diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
index e6d5cc75ca3..5845d65889e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
@@ -318,6 +318,12 @@ public boolean isRequestAccessPopupRequired() {
         return FileUtil.isRequestAccessPopupRequired(fileMetadata.getDatasetVersion());
     }
 
+    public boolean isGuestbookAndTermsPopupRequired() {  
+        if(fileMetadata.getId() == null || fileMetadata.getDatasetVersion().getId() == null ){
+            return false;
+        }
+        return FileUtil.isGuestbookAndTermsPopupRequired(fileMetadata.getDatasetVersion());
+    }
 
     public void setFileMetadata(FileMetadata fileMetadata) {
         this.fileMetadata = fileMetadata;
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 69404482fce..5c39d1039d9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -65,8 +65,12 @@ public class GuestbookResponse implements Serializable {
     @JoinColumn(nullable=true)
     private AuthenticatedUser authenticatedUser;
 
-    @OneToOne(cascade=CascadeType.ALL,mappedBy="guestbookResponse",fetch = FetchType.LAZY, optional = false)
+    @OneToOne(cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},mappedBy="guestbookResponse",fetch = FetchType.LAZY)
     private FileDownload fileDownload;
+
+    @OneToMany(mappedBy="guestbookResponse",cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},fetch = FetchType.LAZY)
+    //private FileAccessRequest fileAccessRequest;
+    private List<FileAccessRequest> fileAccessRequests;
      
     @OneToMany(mappedBy="guestbookResponse",cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},orphanRemoval=true)
     @OrderBy ("id")
@@ -253,6 +257,13 @@ public void setFileDownload(FileDownload fDownload){
         this.fileDownload = fDownload;
     }
     
+    public List<FileAccessRequest> getFileAccessRequests(){
+        return fileAccessRequests;
+    }
+
+    public void setFileAccessRequest(List<FileAccessRequest> fARs){
+        this.fileAccessRequests = fARs;
+    }
     
     public Dataset getDataset() {
         return dataset;
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
index e71e04bc42f..173af4c241f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
@@ -56,6 +56,8 @@ public class ManagePermissionsPage implements java.io.Serializable {
     @EJB
     DvObjectServiceBean dvObjectService;
     @EJB
+    FileAccessRequestServiceBean fileAccessRequestService;
+    @EJB
     DataverseRoleServiceBean roleService;
     @EJB
     RoleAssigneeServiceBean roleAssigneeService;
@@ -400,6 +402,51 @@ public List<RoleAssignee> completeRoleAssignee( String query ) {
         return roleAssigneeService.filterRoleAssignees(query, dvObject, roleAssignSelectedRoleAssignees);
     }
 
+    public void grantAccess(ActionEvent evt) {
+        //QDRADA 
+        logger.info("grantAccess Called");
+        try {
+        throw new Exception("grantAccessCalled");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        /*
+        // Find the built in file downloader role (currently by alias)
+        DataverseRole fileDownloaderRole = roleService.findBuiltinRoleByAlias(DataverseRole.FILE_DOWNLOADER);
+        for (RoleAssignee roleAssignee : selectedRoleAssignees) {
+            boolean sendNotification = false;
+            for (DataFile file : selectedFiles) {
+                if (assignRole(roleAssignee, file, fileDownloaderRole)) {
+                    if (file.isReleased()) {
+                        sendNotification = true;
+                    }
+                    // remove request, if it exist
+                    for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
+                            if (file.getFileAccessRequesters().remove(au)) {
+                                List<FileAccessRequest> fileAccessRequests = fileAccessRequestService.findAllByAuthenticatedUserIdAndRequestState(au.getId(), FileAccessRequest.RequestState.CREATED);
+                                for(FileAccessRequest far : fileAccessRequests){
+                                    far.setStateGranted();
+                                    fileAccessRequestService.save(far);
+                                }
+                                file.setFileAccessRequests(fileAccessRequests); 
+                                datafileService.save(file);
+                            }       
+                    }
+                }
+
+            }
+
+            if (sendNotification) {
+                for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
+                    userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.GRANTFILEACCESS, dataset.getId());                
+                }
+             }
+        }
+
+        initMaps();
+        */
+    }
+    
     public List<DataverseRole> getAvailableRoles() {
         List<DataverseRole> roles = new LinkedList<>();
         if (dvObject != null && dvObject.getId() != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java
index b68a1b9d13e..c91f7630caa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/UserNotification.java
+++ b/src/main/java/edu/harvard/iq/dataverse/UserNotification.java
@@ -39,7 +39,7 @@ public enum Type {
         CHECKSUMIMPORT, CHECKSUMFAIL, CONFIRMEMAIL, APIGENERATED, INGESTCOMPLETED, INGESTCOMPLETEDWITHERRORS, 
         PUBLISHFAILED_PIDREG, WORKFLOW_SUCCESS, WORKFLOW_FAILURE, STATUSUPDATED, DATASETCREATED, DATASETMENTIONED,
         GLOBUSUPLOADCOMPLETED, GLOBUSUPLOADCOMPLETEDWITHERRORS,
-        GLOBUSDOWNLOADCOMPLETED, GLOBUSDOWNLOADCOMPLETEDWITHERRORS;
+        GLOBUSDOWNLOADCOMPLETED, GLOBUSDOWNLOADCOMPLETEDWITHERRORS, REQUESTEDFILEACCESS;
         
         public String getDescription() {
             return BundleUtil.getStringFromBundle("notification.typeDescription." + this.name());
diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java
index 947ee3ce989..972f26f6830 100644
--- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java
@@ -131,6 +131,28 @@ public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate
             save(userNotification);
         }
     }
+    
+    /**
+     * Returns a UserNotification that was sent to a dataverseUser.
+     * Sends ONLY the UserNotification (no email is sent via this method).
+     * All parameters are assumed to be valid, non-null objects.
+     * 
+     * @param dataverseUser - the AuthenticatedUser to whom the notification is to be sent
+     * @param sendDate - the time and date the notification was sent.
+     * @param type - the type of notification to be sent (see UserNotification for the different types)
+     * @param objectId -  the ID of the Dataverse object (Dataverse, Dataset, etc.) that the notification pertains to 
+     * @return The UserNotification that was sent to the dataverseUser
+     */
+
+    public UserNotification sendUserNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId) {
+        UserNotification userNotification = new UserNotification();
+        userNotification.setUser(dataverseUser);
+        userNotification.setSendDate(sendDate);
+        userNotification.setType(type);
+        userNotification.setObjectId(objectId);
+        this.save(userNotification);
+        return userNotification;
+    }
 
     public boolean isEmailMuted(UserNotification userNotification) {
         final Type type = userNotification.getType();
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java
index 9fdfce2f1a7..bb688fb8acb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java
@@ -1,7 +1,9 @@
 package edu.harvard.iq.dataverse.authorization.users;
 
 import edu.harvard.iq.dataverse.Cart;
+import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DatasetLock;
+import edu.harvard.iq.dataverse.FileAccessRequest;
 import edu.harvard.iq.dataverse.UserNotification.Type;
 import edu.harvard.iq.dataverse.UserNotification;
 import edu.harvard.iq.dataverse.validation.ValidateEmail;
@@ -17,6 +19,7 @@
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import java.io.Serializable;
 import java.sql.Timestamp;
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
@@ -28,6 +31,7 @@
 import javax.persistence.CascadeType;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.FetchType;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
@@ -202,6 +206,27 @@ public void setDatasetLocks(List<DatasetLock> datasetLocks) {
     @OneToMany(mappedBy = "user", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
     private List<OAuth2TokenData> oAuth2TokenDatas;
 
+    /*for many to many fileAccessRequests*/
+    @OneToMany(mappedBy = "user", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REFRESH}, fetch = FetchType.LAZY)
+    private List<FileAccessRequest> fileAccessRequests;
+
+    public List<FileAccessRequest> getFileAccessRequests() {
+        return fileAccessRequests;
+    }
+
+    public void setFileAccessRequests(List<FileAccessRequest> fARs) {
+        this.fileAccessRequests = fARs;
+    }
+
+    public List<DataFile> getRequestedDataFiles(){
+        List<DataFile> requestedDataFiles = new ArrayList<>();
+
+        for(FileAccessRequest far : getFileAccessRequests()){
+            requestedDataFiles.add(far.getDataFile());
+        }
+        return requestedDataFiles;
+    }
+    
     @Override
     public AuthenticatedUserDisplayInfo getDisplayInfo() {
         return new AuthenticatedUserDisplayInfo(firstName, lastName, email, affiliation, position);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index d710ed66551..f6a3b287778 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -5,7 +5,11 @@
  */
 package edu.harvard.iq.dataverse.engine.command.impl;
 
+import java.util.logging.Logger;
+
 import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.FileAccessRequest;
+import edu.harvard.iq.dataverse.GuestbookResponse;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
@@ -23,8 +27,11 @@
 @RequiredPermissions({})
 public class RequestAccessCommand extends AbstractCommand<DataFile> {
     
+    private static final Logger logger = Logger.getLogger(RequestAccessCommand.class.getName());
+    
     private final DataFile file;
     private final AuthenticatedUser requester;
+    private final FileAccessRequest fileAccessRequest;
     private final Boolean sendNotification;
 
 
@@ -33,6 +40,7 @@ public RequestAccessCommand(DataverseRequest dvRequest, DataFile file) {
         super(dvRequest, file);        
         this.file = file;        
         this.requester = (AuthenticatedUser) dvRequest.getUser();
+        this.fileAccessRequest = new FileAccessRequest(file,requester);
         this.sendNotification = false;
     }
     
@@ -41,8 +49,27 @@ public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean s
         super(dvRequest, file);        
         this.file = file;        
         this.requester = (AuthenticatedUser) dvRequest.getUser();
+        this.fileAccessRequest = new FileAccessRequest(file,requester);
         this.sendNotification = sendNotification;
     }
+        
+        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) {
+            // for data file check permission on owning dataset
+            super(dvRequest, file);        
+            this.file = file;        
+            this.requester = (AuthenticatedUser) dvRequest.getUser();
+            this.fileAccessRequest = new FileAccessRequest(file,requester,gbr);
+            this.sendNotification = false;
+        }
+
+        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, Boolean sendNotification) {
+            // for data file check permission on owning dataset
+            super(dvRequest, file);        
+            this.file = file;
+            this.requester = (AuthenticatedUser) dvRequest.getUser();
+            this.fileAccessRequest = new FileAccessRequest(file,requester,gbr);
+            this.sendNotification = sendNotification;
+        }
 
         
         public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) {
@@ -75,9 +102,13 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         if(FileUtil.isActivelyEmbargoed(file)) {
             throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.embargoed"), this);
         }
+        file.getFileAccessRequests().add(fileAccessRequest);
         file.addFileAccessRequester(requester);
+        requester.getFileAccessRequests().add(fileAccessRequest);
         if (sendNotification) {
-            ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);
+            //QDRADA
+            logger.info("ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);"); 
+            //ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);
         }
         return ctxt.files().save(file);
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 6bb7e1d583b..a5fb98f7c49 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -28,6 +28,7 @@
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.Embargo;
 import edu.harvard.iq.dataverse.FileMetadata;
+import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.S3AccessIO;
@@ -1639,6 +1640,71 @@ private static Boolean popupDueToStateOrTerms(DatasetVersion datasetVersion) {
         return null;
     }
 
+    /**
+     * isGuestbookAndTermsPopupRequired
+     * meant to replace both isDownloadPopupRequired() and isRequestAccessDownloadPopupRequired() when the guestbook-terms-popup-fragment.xhtml
+     * replaced file-download-popup-fragment.xhtml and file-request-access-popup-fragment.xhtml
+     * @param datasetVersion
+     * @return boolean
+     */
+
+    public static boolean isGuestbookAndTermsPopupRequired(DatasetVersion datasetVersion) {
+        return isGuestbookPopupRequired(datasetVersion) || isTermsPopupRequired(datasetVersion);
+    }
+
+    public static boolean isGuestbookPopupRequired(DatasetVersion datasetVersion) {
+
+        if (datasetVersion == null) {
+            logger.fine("GuestbookPopup not required because datasetVersion is null.");
+            return false;
+        }
+        //0. if version is draft then Popup "not required"
+        if (!datasetVersion.isReleased()) {
+            logger.fine("GuestbookPopup not required because datasetVersion has not been released.");
+            return false;
+        }
+
+        // 3. Guest Book:
+        if (datasetVersion.getDataset() != null && datasetVersion.getDataset().getGuestbook() != null && datasetVersion.getDataset().getGuestbook().isEnabled() && datasetVersion.getDataset().getGuestbook().getDataverse() != null) {
+            logger.fine("GuestbookPopup required because an enabled guestbook exists.");
+            return true;
+        }
+
+        logger.fine("GuestbookPopup is not required.");
+        return false;
+    }
+
+    public static boolean isTermsPopupRequired(DatasetVersion datasetVersion) {
+
+        if (datasetVersion == null) {
+            logger.fine("TermsPopup not required because datasetVersion is null.");
+            return false;
+        }
+        //0. if version is draft then Popup "not required"
+        if (!datasetVersion.isReleased()) {
+            logger.fine("TermsPopup not required because datasetVersion has not been released.");
+            return false;
+        }
+        // 1. License and Terms of Use:
+        if (datasetVersion.getTermsOfUseAndAccess() != null) {
+            if (!License.CC0.equals(datasetVersion.getTermsOfUseAndAccess().getLicense())
+                    && !(datasetVersion.getTermsOfUseAndAccess().getTermsOfUse() == null
+                    || datasetVersion.getTermsOfUseAndAccess().getTermsOfUse().equals(""))) {
+                logger.fine("TermsPopup required because of license or terms of use.");
+                return true;
+            }
+
+            // 2. Terms of Access:
+            if (!(datasetVersion.getTermsOfUseAndAccess().getTermsOfAccess() == null) && !datasetVersion.getTermsOfUseAndAccess().getTermsOfAccess().equals("")) {
+                logger.fine("TermsPopup required because of terms of access.");
+                return true;
+            }
+        }
+
+        logger.fine("TermsPopup is not required.");
+        return false;
+    }
+    
     /**
      * Provide download URL if no Terms of Use, no guestbook, and not
      * restricted.
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java
index 72980c3451a..d166cc753cc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java
@@ -51,6 +51,8 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti
                 return BundleUtil.getStringFromBundle("notification.email.create.dataverse.subject", rootDvNameAsList);
             case REQUESTFILEACCESS:
                 return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", rootDvNameAsList);
+            case REQUESTEDFILEACCESS:
+                return BundleUtil.getStringFromBundle("notification.email.requested.file.access.subject", rootDvNameAsList);
             case GRANTFILEACCESS:
                 return BundleUtil.getStringFromBundle("notification.email.grant.file.access.subject", rootDvNameAsList);
             case REJECTFILEACCESS:
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 4ba6ad0e7e1..a79e421fa58 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1048,11 +1048,11 @@
                         <div class="button-block">
                             <p class="help-block">#{bundle['dataset.downloadUnrestricted']}</p>
                             <p:commandButton styleClass="btn btn-default" value="#{bundle.continue}" onclick="PF('downloadMixed').hide()" 
-                                             rendered="#{!DatasetPage.downloadPopupRequired}"
+                                             rendered="#{!DatasetPage.guestbookAndTermsPopupRequired}"
                                              action="#{DatasetPage.startMultipleFileDownload()}"/>
                             <p:commandButton styleClass="btn btn-default" value="#{bundle.continue}" onclick="PF('downloadMixed').hide();"
-                                             rendered="#{DatasetPage.downloadPopupRequired and !settingsWrapper.rsyncDownload}"
-                                             oncomplete="PF('downloadPopup').show();" />
+                                             rendered="#{DatasetPage.guestbookAndTermsPopupRequired and !settingsWrapper.rsyncDownload}"
+                                             oncomplete="PF('guestbookAndTermsPopup').show();" />
                             <button class="btn btn-link" onclick="PF('downloadMixed').hide();" type="button">
                                 #{bundle.cancel}
                             </button>
@@ -1494,12 +1494,12 @@
                         </div>
                     </p:dialog>
                     <!-- END: Request Access Sign Up/Log In Button -->
-                    <p:dialog id="downloadPopup" styleClass="largePopUp" header="#{bundle['file.downloadDialog.header']}" widgetVar="downloadPopup" modal="true">
+                    <p:dialog id="guestbookAndTermsPopup" styleClass="largePopUp" header="#{bundle['file.downloadDialog.header']}" widgetVar="guestbookAndTermsPopup" modal="true">
                         <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
-                        <ui:include src="file-download-popup-fragment.xhtml">
+                        <ui:include src="guestbook-terms-popup-fragment.xhtml">
                             <ui:param name="popupContext" value="downloadDataset"/>
                             <ui:param name="workingVersion" value="#{DatasetPage.workingVersion}"/>
-                            <ui:param name="downloadPopupRequired" value="#{DatasetPage.downloadPopupRequired}"/>
+                            <ui:param name="guestbookAndTermsPopupRequired" value="#{DatasetPage.guestbookAndTermsPopupRequired}"/>
                             <ui:param name="fileMetadataForAction" value="#{DatasetPage.fileMetadataForAction}"/>
                             <ui:param name="hasRestrictedFile" value="#{DatasetPage.downloadingRestrictedFiles()}"/>
                             <ui:param name="guestbookResponse" value="#{DatasetPage.guestbookResponse}" rendered="#{!settingsWrapper.rsyncDownload}"/>
@@ -1530,19 +1530,11 @@
                         <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
                         <ui:include src="package-download-popup-fragment.xhtml">
                             <ui:param name="workingVersion" value="#{DatasetPage.workingVersion}"/>
-                            <ui:param name="downloadPopupRequired" value="#{DatasetPage.downloadPopupRequired}"/>
+                            <ui:param name="guestbookAndTermsPopupRequired" value="#{DatasetPage.guestbookAndTermsPopupRequired}"/>
                             <ui:param name="guestbookResponse" value="#{DatasetPage.guestbookResponse}" rendered="#{!settingsWrapper.rsyncDownload}"/>
                             <ui:param name="fileDownloadService" value="#{DatasetPage.fileDownloadService}"/>
                         </ui:include>
                     </p:dialog>
-                    <p:dialog id="requestAccessPopup" styleClass="largePopUp" header="#{bundle['file.requestAccess']}" widgetVar="requestAccessPopup" modal="true">
-                        <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
-                        <ui:include src="file-request-access-popup-fragment.xhtml">
-                            <ui:param name="workingVersion" value="#{DatasetPage.workingVersion}"/>
-                            <ui:param name="someActivelyEmbargoedFiles" value="#{DatasetPage.cantRequestDueToEmbargo}"/>
-                            <ui:param name="fileDownloadService" value="#{DatasetPage.fileDownloadService}"/>
-                        </ui:include>
-                    </p:dialog>
                     <p:dialog id="linkDatasetForm" styleClass="largePopUp" header="#{bundle['dataset.link.title']}" widgetVar="linkDatasetForm" modal="true" rendered="#{DatasetPage.showLinkingPopup}">
                         <p:focus for="dataverseLinkName"/>
                         <div class="form-horizontal">
@@ -1883,7 +1875,7 @@
                             PF('downloadInvalid').show();
                         }
                         if (outcome ==='GuestbookRequired'){
-                            PF('downloadPopup').show();
+                            PF('guestbookAndTermsPopup').show();
                         }
                     }
                     
diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index 597d9a12786..4b075eb3377 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -60,7 +60,7 @@
         <!-- GlobusTransfer, orig file only  -->
          <o:importFunctions type="edu.harvard.iq.dataverse.dataaccess.DataAccess" />
         <li jsf:rendered="#{settingsWrapper.globusFileDownload and settingsWrapper.isGlobusEnabledStorageDriver(DataAccess:getStorageDriverFromIdentifier(fileMetadata.dataFile.storageIdentifier))}">
-            <p:commandLink rendered="#{!(downloadPopupRequired) 
+            <p:commandLink rendered="#{!(guestbookAndTermsPopupRequired) 
                                       and !(fileMetadata.dataFile.filePackage)}"
                          styleClass="btn-download"
                          process="@this"
@@ -70,12 +70,13 @@
                 <!-- no guest book/terms of use/etc. - straight to the download API url: -->
                 <span class="globus-btn ui-icon" title="#{bundle['file.globus.transfer']}"/> #{bundle['file.globus.of']} #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
-            <p:commandLink rendered="#{downloadPopupRequired and !fileMetadata.dataFile.filePackage}"
+            <p:commandLink rendered="#{guestbookAndTermsPopupRequired and !fileMetadata.dataFile.filePackage}"
                          styleClass="btn-download"
                          process="@this"
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                          action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'GlobusTransfer')}"
-                         update="@widgetVar(downloadPopup)" oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                         update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                         <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
                 GT: #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
@@ -85,7 +86,7 @@
         
         <!-- NON-TABULAR + PACKAGE DOWNLOAD OPTIONS -->
         <li jsf:rendered="#{!fileMetadata.dataFile.tabularData}">
-            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and !(downloadPopupRequired) 
+            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and !(guestbookAndTermsPopupRequired) 
                                     and fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload}"
                          styleClass="btn-download" process="@this"
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
@@ -96,19 +97,20 @@
                 <!-- no guest book/terms of use/etc. - straight package popup -->
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
-            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and (downloadPopupRequired) 
+            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and (guestbookAndTermsPopupRequired) 
                                     and fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload}"
                          styleClass="btn-download"
                          process="@this"
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                          action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'package')}"
-                         update="@widgetVar(downloadPopup)" oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                         update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                         <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <f:actionListener binding="#{packagePopupFragmentBean.setFileMetadata(fileMetadata)}" /> 
                 <!-- package data file: -->
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
-            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and !(downloadPopupRequired) 
+            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and !(guestbookAndTermsPopupRequired) 
                                       and !(fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload)}"
                          styleClass="btn-download"
                          process="@this"
@@ -118,13 +120,13 @@
                 <!-- no guest book/terms of use/etc. - straight to the download API url: -->
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
-            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and downloadPopupRequired 
+            <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and guestbookAndTermsPopupRequired 
                                       and !(fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload)}"
                          styleClass="btn-download"
                          process="@this"
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                          action="#{guestbookResponseService.modifyDatafile(guestbookResponse, fileMetadata)}"
-                         update="@widgetVar(downloadPopup)" oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                         update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
                 <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
@@ -134,23 +136,24 @@
         <ui:fragment rendered="#{fileMetadata.dataFile.tabularData}">
             <ui:remove>
             <li>
-                <p:commandLink styleClass="highlightBold btn-download" rendered="#{!(downloadPopupRequired)}"
+                <p:commandLink styleClass="highlightBold btn-download" rendered="#{!(guestbookAndTermsPopupRequired)}"
                                process="@this"
                                actionListener="#{fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse, fileMetadata, 'bundle')}">
                     #{bundle['file.downloadBtn.format.all']}
                 </p:commandLink>
-                <p:commandLink styleClass="highlightBold btn-download" rendered="#{downloadPopupRequired}"
+                <p:commandLink styleClass="highlightBold btn-download" rendered="#{guestbookAndTermsPopupRequired}"
                                process="@this"
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'bundle' )}"
-                               update="@widgetVar(downloadPopup)"
-                               oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                               update="@widgetVar(guestbookAndTermsPopup)"
+                               oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                     #{bundle['file.downloadBtn.format.all']}
                 </p:commandLink>
             </li>
             <li role="presentation" class="divider"></li>
             </ui:remove>
             <li>
-                <p:commandLink styleClass="btn-download" rendered="#{!downloadPopupRequired and !(fileMetadata.dataFile.originalFormatLabel == 'UNKNOWN')}"
+                <p:commandLink styleClass="btn-download" rendered="#{!guestbookAndTermsPopupRequired and !(fileMetadata.dataFile.originalFormatLabel == 'UNKNOWN')}"
                                process="@this"
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                actionListener="#{fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse, fileMetadata, 'original')}">
@@ -158,12 +161,13 @@
                         <f:param value="#{fileMetadata.dataFile.originalFormatLabel}"/>
                     </h:outputFormat>
                 </p:commandLink>
-                <p:commandLink styleClass="btn-download" rendered="#{downloadPopupRequired and !(fileMetadata.dataFile.originalFormatLabel == 'UNKNOWN')}"
+                <p:commandLink styleClass="btn-download" rendered="#{guestbookAndTermsPopupRequired and !(fileMetadata.dataFile.originalFormatLabel == 'UNKNOWN')}"
                                process="@this"
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'original' )}"
-                               update="@widgetVar(downloadPopup)"
-                               oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                               update="@widgetVar(guestbookAndTermsPopup)"
+                               oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                     <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                     <h:outputFormat value="#{bundle['file.downloadBtn.format.original']}">
                         <f:param value="#{fileMetadata.dataFile.originalFormatLabel}"/>
@@ -171,35 +175,37 @@
                 </p:commandLink>
             </li>
             <li>
-                <p:commandLink styleClass="btn-download" rendered="#{!downloadPopupRequired}"
+                <p:commandLink styleClass="btn-download" rendered="#{!guestbookAndTermsPopupRequired}"
                                process="@this"
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                actionListener="#{fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse, fileMetadata, 'tab')}">
                     #{bundle['file.downloadBtn.format.tab']}
                 </p:commandLink>
-                <p:commandLink styleClass="btn-download" rendered="#{downloadPopupRequired}"
+                <p:commandLink styleClass="btn-download" rendered="#{guestbookAndTermsPopupRequired}"
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'tab' )}"
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
-                               update="@widgetVar(downloadPopup)"
-                               oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                               update="@widgetVar(guestbookAndTermsPopup)"
+                               oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                     <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                     #{bundle['file.downloadBtn.format.tab']}
                 </p:commandLink>
             </li>
             <ui:fragment rendered="#{!(fileMetadata.dataFile.originalFormatLabel == 'RData')}">
                 <li>
-                    <p:commandLink styleClass="btn-download" rendered="#{!downloadPopupRequired}"
+                    <p:commandLink styleClass="btn-download" rendered="#{!guestbookAndTermsPopupRequired}"
                                    process="@this"
                                    disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                    actionListener="#{fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse, fileMetadata, 'RData')}">
                         #{bundle['file.downloadBtn.format.rdata']}
                     </p:commandLink>
-                    <p:commandLink styleClass="btn-download" rendered="#{downloadPopupRequired}"
+                    <p:commandLink styleClass="btn-download" rendered="#{guestbookAndTermsPopupRequired}"
                                    process="@this"
                                    disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                    action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'RData' )}"
-                                   update="@widgetVar(downloadPopup)"
-                                   oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                                   update="@widgetVar(guestbookAndTermsPopup)"
+                                   oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                                   <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                         <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                         #{bundle['file.downloadBtn.format.rdata']}
                     </p:commandLink>
@@ -215,18 +221,19 @@
     
     <ui:fragment rendered="#{fileMetadata.dataFile.tabularData and fileDownloadHelper.canDownloadFile(fileMetadata)}">
         <li>
-            <p:commandLink styleClass="btn-download" rendered="#{!downloadPopupRequired}"
+            <p:commandLink styleClass="btn-download" rendered="#{!guestbookAndTermsPopupRequired}"
                            process="@this"
                            disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                            actionListener="#{fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse, fileMetadata, 'var')}">
                 #{bundle['file.downloadBtn.format.var']}
             </p:commandLink>
-            <p:commandLink styleClass="btn-download" rendered="#{downloadPopupRequired}"
+            <p:commandLink styleClass="btn-download" rendered="#{guestbookAndTermsPopupRequired}"
                            process="@this"
                            disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                            action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'var' )}"
-                           update="@widgetVar(downloadPopup)"
-                           oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                           update="@widgetVar(guestbookAndTermsPopup)"
+                           oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                           <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                 #{bundle['file.downloadBtn.format.var']}
             </p:commandLink>
         </li>
@@ -303,20 +310,21 @@
         <!-- externalTools -->
         <ui:repeat var="tool" value="#{exploreTools}">
             <li>
-                <p:commandLink rendered="#{!downloadPopupRequired}"
+                <p:commandLink rendered="#{!guestbookAndTermsPopupRequired}"
                                styleClass="btn-explore #{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
                                action="#{fileDownloadService.explore(guestbookResponse, fileMetadata, tool )}">
                     #{tool.getDisplayNameLang()}
                 </p:commandLink>
                 <!--The modifyDatafileAndFormat method below was added because on the dataset page, "tool" is null in the popup so we store it in the guestbookResponse because we know we'll need it later in the popup.-->
-                <p:commandLink rendered="#{downloadPopupRequired}"
+                <p:commandLink rendered="#{guestbookAndTermsPopupRequired}"
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'externalTool', tool)}"
                                styleClass="btn-explore #{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload)}"
                                process="@this"
-                               update="@widgetVar(downloadPopup)"
-                               oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                               update="@widgetVar(guestbookAndTermsPopup)"
+                               oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                     #{tool.getDisplayNameLang()}
                 </p:commandLink>
             </li>
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index ae8729fdf89..d27536cb892 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -196,7 +196,7 @@
                                                                                    or FilePage.fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload}">
                                                             <ui:include src="file-download-button-fragment.xhtml">
                                                                 <ui:param name="fileMetadata" value="#{FilePage.fileMetadata}"/>
-                                                                <ui:param name="downloadPopupRequired" value="#{FilePage.downloadPopupRequired}"/>
+                                                                <ui:param name="guestbookAndTermsPopupRequired" value="#{FilePage.guestbookAndTermsPopupRequired}"/>
                                                                 <ui:param name="requestAccessPopupRequired" value="#{FilePage.requestAccessPopupRequired}"/>
                                                                 <ui:param name="guestbookResponse" value="#{FilePage.guestbookResponse}"/>
                                                                 <ui:param name="guestbookResponseService" value="#{FilePage.guestbookResponseService}"/>
@@ -343,11 +343,11 @@
                             <p:tab id="previewTab" title="#{bundle['file.previewTab.header']}" 
                                    rendered="#{FilePage.toolsWithPreviews.size() > 0 and fileDownloadHelper.canDownloadFile(FilePage.fileMetadata)}">
                                 <!-- PREVIEW TERMS/GUESTBOOK FORM -->
-                                <ui:fragment rendered="#{FilePage.downloadPopupRequired and !FilePage.termsMet}">
+                                <ui:fragment rendered="#{FilePage.guestbookAndTermsPopupRequired and !FilePage.termsMet}">
                                     <ui:include src="file-download-popup-fragment.xhtml">
                                         <ui:param name="popupContext" value="previewTab"/>
                                         <ui:param name="workingVersion" value="#{FilePage.fileMetadata.datasetVersion}"/>
-                                        <ui:param name="downloadPopupRequired" value="#{FilePage.downloadPopupRequired}"/>
+                                        <ui:param name="guestbookAndTermsPopupRequired" value="#{FilePage.guestbookAndTermsPopupRequired}"/>
                                         <ui:param name="hasRestrictedFile" value="#{FilePage.fileMetadata.restricted}"/>
                                         <ui:param name="guestbookResponse" value="#{FilePage.guestbookResponse}"/>
                                         <ui:param name="guestbookResponseService" value="#{FilePage.guestbookResponseService}"/>
@@ -356,7 +356,7 @@
                                     </ui:include>
                                 </ui:fragment>
                                 <!-- PREVIEW EXTERNAL TOOL -->
-                                <ui:fragment rendered="#{(!FilePage.downloadPopupRequired) or (FilePage.downloadPopupRequired and FilePage.termsMet)}">
+                                <ui:fragment rendered="#{(!FilePage.guestbookAndTermsPopupRequired) or (FilePage.guestbookAndTermsPopupRequired and FilePage.termsMet)}">
                                 <div class="btn-toolbar margin-bottom" role="toolbar" aria-label="#{bundle['file.previewTab.button.label']}">
                                     <!-- Preview Button Group -->
                                     <div class="btn-group" jsf:rendered="#{FilePage.toolsWithPreviews.size() > 1 and fileDownloadHelper.canDownloadFile(FilePage.fileMetadata)}">
@@ -376,7 +376,7 @@
                                     <!-- END: Preview Button Group -->
                                     <div class="btn-group" jsf:rendered="#{FilePage.toolsWithPreviews.size() > 0 and fileDownloadHelper.canDownloadFile(FilePage.fileMetadata)}">
                                         <!-- Modular/Configured Explore Tool -->
-                                        <h:commandLink rendered="#{!downloadPopupRequired}" 
+                                        <h:commandLink rendered="#{!guestbookAndTermsPopupRequired}" 
                                                          type="submit"
                                                          styleClass="btn btn-default #{(FilePage.fileMetadata.dataFile.ingestInProgress) ? 'disabled' : ''}"
                                                          disabled="#{(FilePage.fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
@@ -387,14 +387,14 @@
                                                 <f:param value="#{FilePage.selectedTool.getDisplayNameLang()}"/>
                                             </h:outputFormat>
                                         </h:commandLink>
-                                        <p:commandLink rendered="#{downloadPopupRequired}"
+                                        <p:commandLink rendered="#{guestbookAndTermsPopupRequired}"
                                                          action="#{FilePage.guestbookResponseService.modifyDatafileAndFormat(FilePage.guestbookResponse, FilePage.fileMetadata, 'externalTool', FilePage.selectedTool)}"
                                                          styleClass="btn btn-default"
                                                          disabled="#{(FilePage.fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
                                                          type="submit"
                                                          process="@this"
-                                                         update="@widgetVar(downloadPopup)"
-                                                         oncomplete="PF('downloadPopup').show();handleResizeDialog('downloadPopup');">
+                                                         update="@widgetVar(guestbookAndTermsPopup)"
+                                                         oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
                                             <span class="glyphicon glyphicon-#{FilePage.selectedTool.exploreTool ? 'equalizer' : 'new-window'}"></span>
                                             <h:outputFormat value="#{FilePage.selectedTool.exploreTool ? bundle['file.previewTab.exploreBtn'] : bundle['file.previewTab.openBtn']}">
                                                 <f:param value="#{bundle.explore}"/>
@@ -616,36 +616,28 @@
                             </button>
                         </div>
                     </p:dialog>                                                              
-                    <p:dialog id="downloadPopup" styleClass="largePopUp" header="#{bundle['file.downloadDialog.header']}" widgetVar="downloadPopup" modal="true">
-                         <ui:include src="file-download-popup-fragment.xhtml">
-                            <ui:param name="popupContext" value="downloadFile"/>
-                            <ui:param name="workingVersion" value="#{FilePage.fileMetadata.datasetVersion}"/>
-                            <ui:param name="downloadPopupRequired" value="#{FilePage.downloadPopupRequired}"/>
-                            <ui:param name="hasRestrictedFile" value="#{FilePage.fileMetadata.restricted}"/>
-                            <ui:param name="guestbookResponse" value="#{FilePage.guestbookResponse}"/>
-                            <ui:param name="guestbookResponseService" value="#{FilePage.guestbookResponseService}"/>
-                            <ui:param name="fileDownloadService" value="#{FilePage.fileDownloadService}"/>
-                            <ui:param name="lockedFromDownload" value="#{FilePage.lockedFromDownload}"/>
-                        </ui:include>
-                    </p:dialog>
+
                     <p:dialog id="downloadPackagePopup" styleClass="smallPopUp" header="#{bundle['packageDownload.title']}" widgetVar="downloadPackagePopup" modal="true">
                          <ui:include src="package-download-popup-fragment.xhtml">
                             <ui:param name="workingVersion" value="#{FilePage.fileMetadata.datasetVersion}"/>
-                            <ui:param name="downloadPopupRequired" value="#{FilePage.downloadPopupRequired}"/>
+                            <ui:param name="guestbookAndTermsPopupRequired" value="#{FilePage.guestbookAndTermsPopupRequired}"/>
                             <ui:param name="guestbookResponse" value="#{FilePage.guestbookResponse}"/>
                             <ui:param name="guestbookResponseService" value="#{FilePage.guestbookResponseService}"/>
                             <ui:param name="fileDownloadService" value="#{FilePage.fileDownloadService}"/>
                             <ui:param name="lockedFromDownload" value="#{FilePage.lockedFromDownload}"/>
                         </ui:include>
                     </p:dialog>                  
-                    <p:dialog id="requestAccessPopup" styleClass="largePopUp" header="#{bundle['file.requestAccess']}" widgetVar="requestAccessPopup" modal="true">
+                    <p:dialog id="guestbookAndTermsPopup" styleClass="largePopUp" header="#{bundle['file.downloadDialog.header']}" widgetVar="guestbookAndTermsPopup" modal="true">
                         <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
-                        <ui:include src="file-request-access-popup-fragment.xhtml">
+                        <ui:include src="guestbook-terms-popup-fragment.xhtml">
                             <ui:param name="workingVersion" value="#{FilePage.fileMetadata.datasetVersion}"/>
                             <ui:param name="someActivelyEmbargoedFiles" value="#{FilePage.cantRequestDueToEmbargo}"/>
+                            <ui:param name="guestbookAndTermsPopupRequired" value="#{FilePage.guestbookAndTermsPopupRequired}"/>
+                            <ui:param name="guestbookResponse" value="#{FilePage.guestbookResponse}"/>
+                            <ui:param name="fileDownloadHelper" value="#{FilePage.fileDownloadHelper}"/>
                             <ui:param name="fileDownloadService" value="#{FilePage.fileDownloadService}"/>
                         </ui:include>
-                    </p:dialog>                    
+                    </p:dialog>
                     <p:dialog id="mapDataDialog" header="#{bundle['file.mapData.unpublished.header']}" widgetVar="mapDataDialog" modal="true">
                         <p class="help-block">
                             <span class="text-danger"><span class="glyphicon glyphicon-exclamation-sign"/> #{bundle['file.mapData.unpublished.message']}</span>
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 7395998042f..fa49936591c 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -472,6 +472,7 @@
                                            update="@form" oncomplete="showPopup();"
                                            onclick="if (!testFilesSelected()) return false;" 
                                            actionListener="#{DatasetPage.startDownloadSelectedArchival()}">
+                            <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                             <f:setPropertyActionListener target="#{DatasetPage.fileMetadataForAction}" value="#{null}"/>
                                 #{bundle.downloadArchival}
                             </p:commandLink>
@@ -482,7 +483,8 @@
                 <p:commandLink rendered="#{DatasetPage.fileAccessRequestMultiButtonRequired}"
                                styleClass="btn btn-default btn-request"                                   
                                action="#{DatasetPage.requestAccessMultipleFiles()}"
-                               update="@form, @([id$=messagePanel])"
+                               update="@form, @([id$=messagePanel])" 
+                               action="#{DatasetPage.validateFilesForRequestAccess()}"
                                disabled="#{DatasetPage.locked or !DatasetPage.fileAccessRequestMultiButtonEnabled}">
                     <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle['file.requestAccess'])}"/>
                     <span class="glyphicon glyphicon-bullhorn"/> #{bundle['file.requestAccess']}
@@ -543,10 +545,11 @@
                         <!-- Explore/Download/Request Button Block -->
                         <ui:include src="file-download-button-fragment.xhtml">
                             <ui:param name="fileMetadata" value="#{fileMetadata}"/>
-                            <ui:param name="downloadPopupRequired" value="#{DatasetPage.downloadPopupRequired}"/>
                             <ui:param name="fileMetadataForAction" value="#{DatasetPage.fileMetadataForAction}"/>
-                            <ui:param name="requestAccessPopupRequired" value="#{DatasetPage.requestAccessPopupRequired}"/>
+                            <ui:param name="termsPopupRequired" value="#{DatasetPage.termsPopupRequired}"/>
+                            <ui:param name="guestbookPopupRequiredAtDownload" value="#{DatasetPage.guestbookPopupRequiredAtDownload}"/>        
                             <ui:param name="guestbookResponse" value="#{DatasetPage.guestbookResponse}"/>
+                            <ui:param name="guestbookAndTermsPopupRequired" value="#{DatasetPage.guestbookAndTermsPopupRequired}"/>
                             <ui:param name="guestbookResponseService" value="#{DatasetPage.guestbookResponseService}"/>
                             <ui:param name="fileDownloadService" value="#{DatasetPage.fileDownloadService}"/>
                             <ui:param name="lockedFromDownload" value="#{DatasetPage.lockedFromDownload}"/>
diff --git a/src/main/webapp/file-request-access-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
similarity index 65%
rename from src/main/webapp/file-request-access-popup-fragment.xhtml
rename to src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 6541d86b686..5616fa48280 100644
--- a/src/main/webapp/file-request-access-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -50,4 +50,21 @@
                 #{bundle.cancel}
             </button>
         </div>
+        <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle['file.requestAccess']}">
+            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
+                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"                          
+                    update="@([id$=Messages])"
+                    oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
+                    <f:param name="DO_GB_VALIDATION" value="true"/>
+            </p:commandLink>
+
+            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{!workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
+                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(fileDownloadHelper.guestbookResponse)}"                          
+                    update="@([id$=Messages])"
+                    oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
+                    <f:param name="DO_GB_VALIDATION" value="true"/>
+            </p:commandLink>
+
+            <button type="button" class="btn btn-default" onclick="PF('guestbookAndTermsPopup').hide();PF('blockDatasetForm').hide();" value="#{bundle.cancel}">#{bundle.cancel}</button>
+        </div>
 </ui:composition>
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java
index 1d481f18cf5..34176f7fb26 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java
@@ -138,7 +138,7 @@ public void testIsDownloadPopupRequiredLicenseCC0() {
             DatasetVersion dsv1 = new DatasetVersion();
             dsv1.setVersionState(DatasetVersion.VersionState.RELEASED);
             TermsOfUseAndAccess termsOfUseAndAccess = new TermsOfUseAndAccess();
-            License license = new License("CC0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true, 1l);
+            License license = new License("CC0 1.0", "Creative Commons CC0 1.0 Universal Public Domain Dedication.", URI.create("https://creativecommons.org/publicdomain/zero/1.0"), URI.create("https://licensebuttons.net/p/zero/1.0/88x31.png"), true,1L);
             license.setDefault(true);
             termsOfUseAndAccess.setLicense(license);
             dsv1.setTermsOfUseAndAccess(termsOfUseAndAccess);

From 0ce8d1cc68d8f4a7c814c477c0f53068648c2ce1 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 15:48:31 -0500
Subject: [PATCH 076/922] try fix - use merged dataset

---
 .../command/impl/CuratePublishedDatasetVersionCommand.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
index b18902b721b..e86b63c701b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
@@ -91,7 +91,9 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         // we have to merge to update the database but not flush because
         // we don't want to create two draft versions!
         Dataset tempDataset = ctxt.em().merge(getDataset());
-
+        
+        updateVersion = tempDataset.getLatestVersionForCopy();
+        
         // Look for file metadata changes and update published metadata if needed
         List<FileMetadata> pubFmds = updateVersion.getFileMetadatas();
         int pubFileCount = pubFmds.size();

From 6e7ab0003314e7b0892612a25b7d6d3ff8c0377e Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 16:08:40 -0500
Subject: [PATCH 077/922] add id column in fileaccessrequest

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 1ffc87dfa32..02543e448c7 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1 +1,2 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT PRIMARY_KEY;

From 16b4c3c69b7e69b0e773d3a7292abe4da04ddef6 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 16:22:26 -0500
Subject: [PATCH 078/922] add constraint directly

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 02543e448c7..7fc270d0822 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1,2 +1,3 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT PRIMARY_KEY;
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT;
+ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
\ No newline at end of file

From 3658b8ec4e49ad7dae7daa9501dc89e0656c07e4 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 17:17:28 -0500
Subject: [PATCH 079/922] remove old constraint, update id col

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 7fc270d0822..864e17de4fd 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1,3 +1,5 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT;
+ALTER TABLE fileaccessrequests DROP CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (datafile_id, authenticated_user_id);
+UPDATE fileaccessrequests SET id = DEFAULT;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
\ No newline at end of file

From 6fda50c5ad3a39d98b6d0e1c38096c58d708a28e Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 17:27:33 -0500
Subject: [PATCH 080/922] syntax err

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 864e17de4fd..e7397e81625 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1,5 +1,5 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT;
-ALTER TABLE fileaccessrequests DROP CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (datafile_id, authenticated_user_id);
+ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
 UPDATE fileaccessrequests SET id = DEFAULT;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
\ No newline at end of file

From b858cd325e1b933a49b91c39d68b4620fe2b2c93 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 17:52:37 -0500
Subject: [PATCH 081/922] another variant

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index e7397e81625..4ddd353e9f9 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1,5 +1,5 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
-UPDATE fileaccessrequests SET id = DEFAULT;
+ALTER TABLE fileaccessrequests ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
\ No newline at end of file

From 3ec1451d2e187ec344e923d77880a9fa90b42234 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Tue, 6 Dec 2022 18:11:52 -0500
Subject: [PATCH 082/922] try try again

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 4ddd353e9f9..a6786db443a 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1,5 +1,4 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id INT;
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
-ALTER TABLE fileaccessrequests ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
\ No newline at end of file

From 76448e7b8fcaca4ad6e25e453189e75d610b133c Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Wed, 7 Dec 2022 09:49:28 -0500
Subject: [PATCH 083/922] add fk constraint

---
 .../db/migration/V5.13.0.3__guestbook-on-request.sql          | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index a6786db443a..8efef213fac 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -1,4 +1,6 @@
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
-ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
\ No newline at end of file
+ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbook_id;
+ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookrequest FOREIGN KEY (guestbookrequest_id) REFERENCES guestbookrequest(id);

From cdc669bdf7b40acb9f6aef04fef4a557765873ee Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Wed, 7 Dec 2022 10:05:02 -0500
Subject: [PATCH 084/922] typos

---
 .../db/migration/V5.13.0.3__guestbook-on-request.sql          | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 8efef213fac..7c3c48bc08a 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -2,5 +2,5 @@ ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbook_id;
-ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookrequest FOREIGN KEY (guestbookrequest_id) REFERENCES guestbookrequest(id);
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id;
+ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);

From 0ed931c849c6e019f3e92a718bf8929ebcd55c15 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Wed, 7 Dec 2022 10:15:15 -0500
Subject: [PATCH 085/922] missing INT

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 7c3c48bc08a..f28e7ca8896 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -2,5 +2,5 @@ ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id;
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);

From 7eb0c5707c1e66484fedcb3be8fe95c83b9e4a73 Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Wed, 7 Dec 2022 10:32:39 -0500
Subject: [PATCH 086/922] add missing methods

---
 .../edu/harvard/iq/dataverse/DatasetPage.java    | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 8754c26a3b6..05a6e6db43c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3190,10 +3190,10 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
 
         
 //QDRADA handle new state from
-        /*if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()){
+        if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()){
             PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
         }
-        */
+        
         
         // If some of the files were restricted and we had to drop them off the
         // list, and NONE of the files are left on the downloadable list
@@ -5310,6 +5310,18 @@ public boolean isGuestbookAndTermsPopupRequired() {
         return FileUtil.isGuestbookAndTermsPopupRequired(workingVersion);
     }
 
+    public boolean isGuestbookPopupRequired(){
+        return FileUtil.isGuestbookPopupRequired(workingVersion);
+    }
+    
+    public boolean isTermsPopupRequired(){
+        return FileUtil.isTermsPopupRequired(workingVersion);
+    }
+    
+    public boolean isGuestbookPopupRequiredAtDownload(){
+        return isGuestbookPopupRequired() && !workingVersion.getDataset().isFileAccessRequest(); //only show guestbookAtDwonload if there is no possible request access
+    }
+    
     public String requestAccessMultipleFiles() {
 
         if (selectedFiles.isEmpty()) {

From 7efe254b1e07396dc9a3d86a304006ec39a6742d Mon Sep 17 00:00:00 2001
From: Jim Myers <myersjd@umich.edu>
Date: Thu, 22 Dec 2022 16:46:55 -0500
Subject: [PATCH 087/922] guestbookresponse /datasetVersion fix

---
 .../edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
index f4cf38979c5..8f887775064 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
@@ -640,6 +640,9 @@ public GuestbookResponse initGuestbookResponseForFragment(DatasetVersion working
        
         GuestbookResponse guestbookResponse = new GuestbookResponse();
         
+        //Not otherwise set for multi-file downloads
+        guestbookResponse.setDatasetVersion(workingVersion);
+        
         if(workingVersion.isDraft()){           
             guestbookResponse.setWriteResponse(false);
         } 

From 2770fc64b40c7c18aab0000499f8a048449a05fb Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 19 May 2023 09:18:42 -0400
Subject: [PATCH 088/922] merge fixes

---
 .../edu/harvard/iq/dataverse/DataFile.java    | 37 ++++++-----
 .../edu/harvard/iq/dataverse/DatasetPage.java | 61 +------------------
 .../iq/dataverse/FileAccessRequest.java       | 21 +++++--
 .../dataverse/ManageFilePermissionsPage.java  |  4 +-
 .../edu/harvard/iq/dataverse/api/Access.java  |  2 +-
 .../command/impl/RequestAccessCommand.java    | 40 ++++--------
 6 files changed, 50 insertions(+), 115 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index c43800c57ce..efceb13092a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -32,8 +32,8 @@
 import javax.json.Json;
 import javax.json.JsonArrayBuilder;
 import javax.persistence.*;
+import javax.validation.constraints.NotBlank;
 import javax.validation.constraints.Pattern;
-import org.hibernate.validator.constraints.NotBlank;
 
 /**
  *
@@ -73,10 +73,6 @@ public class DataFile extends DvObject implements Comparable {
     @Pattern(regexp = "^.*/.*$", message = "{contenttype.slash}")
     private String contentType;
 
-    public void setFileAccessRequests(List<FileAccessRequest> fileAccessRequests) {
-        this.fileAccessRequests = fileAccessRequests;
-    }
-
 //    @Expose    
 //    @SerializedName("storageIdentifier")
 //    @Column( nullable = false )
@@ -203,6 +199,13 @@ public String toString() {
     @OneToMany(mappedBy="dataFile",fetch = FetchType.LAZY,cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST, CascadeType.REFRESH})
     private List<FileAccessRequest> fileAccessRequests;
 
+    @ManyToMany
+    @JoinTable(name = "fileaccessrequests",
+    joinColumns = @JoinColumn(name = "datafile_id"),
+    inverseJoinColumns = @JoinColumn(name = "authenticated_user_id"))
+    private List<AuthenticatedUser> fileAccessRequesters;
+
+    
     public List<FileAccessRequest> getFileAccessRequests(){
         return fileAccessRequests;
     }
@@ -761,20 +764,22 @@ public String getUnf() {
         return null; 
     }
 
-    @OneToMany(mappedBy = "dataFile", cascade = {CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST}, orphanRemoval = true)
-    private List<FileAccessRequest> fileAccessRequests;
+    public List<AuthenticatedUser> getFileAccessRequesters() {
+        return fileAccessRequesters;
+    }
 
-    public List<FileAccessRequest> getFileAccessRequests() {
-        return fileAccessRequests;
+    public void setFileAccessRequesters(List<AuthenticatedUser> fileAccessRequesters) {
+        this.fileAccessRequesters = fileAccessRequesters;
     }
 
+
     public void addFileAccessRequester(AuthenticatedUser authenticatedUser) {
         if (this.fileAccessRequests == null) {
             this.fileAccessRequests = new ArrayList<>();
         }
 
         Set<AuthenticatedUser> existingUsers = this.fileAccessRequests.stream()
-            .map(FileAccessRequest::getAuthenticatedUser)
+            .map(FileAccessRequest::getRequester)
             .collect(Collectors.toSet());
 
         if (existingUsers.contains(authenticatedUser)) {
@@ -784,13 +789,7 @@ public void addFileAccessRequester(AuthenticatedUser authenticatedUser) {
         FileAccessRequest request = new FileAccessRequest();
         request.setCreationTime(new Date());
         request.setDataFile(this);
-        request.setAuthenticatedUser(authenticatedUser);
-
-        FileAccessRequest.FileAccessRequestKey key = new FileAccessRequest.FileAccessRequestKey();
-        key.setAuthenticatedUser(authenticatedUser.getId());
-        key.setDataFile(this.getId());
-
-        request.setId(key);
+        request.setRequester(authenticatedUser);
 
         this.fileAccessRequests.add(request);
     }
@@ -801,7 +800,7 @@ public boolean removeFileAccessRequester(RoleAssignee roleAssignee) {
         }
 
         FileAccessRequest request = this.fileAccessRequests.stream()
-            .filter(fileAccessRequest -> fileAccessRequest.getAuthenticatedUser().equals(roleAssignee))
+            .filter(fileAccessRequest -> fileAccessRequest.getRequester().equals(roleAssignee))
             .findFirst()
             .orElse(null);
 
@@ -819,7 +818,7 @@ public boolean containsFileAccessRequestFromUser(RoleAssignee roleAssignee) {
         }
 
         Set<AuthenticatedUser> existingUsers = this.fileAccessRequests.stream()
-            .map(FileAccessRequest::getAuthenticatedUser)
+            .map(FileAccessRequest::getRequester)
             .collect(Collectors.toSet());
 
         return existingUsers.contains(roleAssignee);
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 05a6e6db43c..c0b4fc104ad 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3127,7 +3127,7 @@ public void startDownloadSelectedOriginal() {
 
     private void startDownload(boolean downloadOriginal){
         boolean guestbookRequired = isDownloadPopupRequired();
-        boolean validate = validateFilesForDownload(guestbookRequired, downloadOriginal);
+        boolean validate = validateFilesForDownload(guestbookRequired);
         if (validate) {
             updateGuestbookResponse(guestbookRequired, downloadOriginal);
             if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){
@@ -3237,65 +3237,6 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
     /*helper function to filter the selected files into <selected downloadable>, 
     and <selected, non downloadable> and <selected restricted> for reuse*/
 
-    private boolean filterSelectedFiles(){
-        setSelectedDownloadableFiles(new ArrayList<>());
-        setSelectedNonDownloadableFiles(new ArrayList<>());
-        setSelectedRestrictedFiles(new ArrayList<>());
-        setSelectedUnrestrictedFiles(new ArrayList<>());
-
-        boolean someFiles = false;
-        for (FileMetadata fmd : this.selectedFiles){
-            if(this.fileDownloadHelper.canDownloadFile(fmd)){
-                getSelectedDownloadableFiles().add(fmd);
-                someFiles=true;
-            } else {
-                getSelectedNonDownloadableFiles().add(fmd);
-            }
-            if(fmd.isRestricted()){
-                getSelectedRestrictedFiles().add(fmd); //might be downloadable to user or not
-                someFiles=true;
-            } else {
-                getSelectedUnrestrictedFiles().add(fmd);
-                someFiles=true;
-            }
-
-        }
-        return someFiles;
-    }
-
-    public void validateFilesForRequestAccess(){
-        this.filterSelectedFiles();
-
-        if(!dataset.isFileAccessRequest()){ //is this needed? wouldn't be able to click Request Access if this !isFileAccessRequest()
-            return;
-        }
-
-        if(!this.selectedRestrictedFiles.isEmpty()){
-            ArrayList nonDownloadableRestrictedFiles = new ArrayList<>();
-
-            List<DataFile> userRequestedDataFiles = ((AuthenticatedUser) session.getUser()).getRequestedDataFiles();
-
-            for(FileMetadata fmd : this.selectedRestrictedFiles){
-                if(!this.fileDownloadHelper.canDownloadFile(fmd) && !userRequestedDataFiles.contains(fmd.getDataFile())){
-                    nonDownloadableRestrictedFiles.add(fmd);
-                }
-            }
-
-            if(!nonDownloadableRestrictedFiles.isEmpty()){
-                guestbookResponse.setDataFile(null);
-                guestbookResponse.setSelectedFileIds(this.getFilesIdsString(nonDownloadableRestrictedFiles));
-
-                if(this.isGuestbookAndTermsPopupRequired()){ //need to pop up the guestbook and terms dialog
-                    PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
-                } else {
-                    this.requestAccessMultipleFiles();
-                }
-            } else {
-                //popup select data files
-            }
-        }
-    }
-
     /*helper function to filter the selected files into <selected downloadable>, 
     and <selected, non downloadable> and <selected restricted> for reuse*/
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
index 723a54c8587..e76a3414b52 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
@@ -1,20 +1,20 @@
 package edu.harvard.iq.dataverse;
 
 import java.io.Serializable;
-import javax.persistence.CascadeType;
+import java.util.Date;
+
 import javax.persistence.GeneratedValue;
 import javax.persistence.UniqueConstraint;
 import javax.persistence.Entity;
 import javax.persistence.Table;
-import javax.persistence.Index;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
 import javax.persistence.Id;
 import javax.persistence.JoinColumn;
-import javax.persistence.JoinTable;
 import javax.persistence.ManyToOne;
 import javax.persistence.OneToOne;
 import javax.persistence.EnumType;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-import edu.harvard.iq.dataverse.datavariable.DataVariable;
 import javax.persistence.Column;
 import javax.persistence.Enumerated;
 import javax.persistence.GenerationType;
@@ -50,6 +50,7 @@
                 query = "SELECT far FROM FileAccessRequest far WHERE far.user.id=:authenticatedUserId and far.dataFile.id=:dataFileId and far.requestState=:requestState")
 })
 
+
 public class FileAccessRequest implements Serializable{
     private static final long serialVersionUID = 1L;
     @Id
@@ -74,6 +75,10 @@ public enum RequestState {CREATED,EDITED,GRANTED,REJECTED,RESUBMIT,INVALIDATED,C
     @Column(name="request_state", nullable=false )
     private RequestState requestState;
     
+    @Temporal(value = TemporalType.TIMESTAMP)
+    @Column(name = "creation_time")
+    private Date creationTime;
+    
     public FileAccessRequest(){
         
     }
@@ -220,6 +225,14 @@ public int hashCode() {
         return hash;
     }
 
+    public Date getCreationTime() {
+        return creationTime;
+    }
+
+    public void setCreationTime(Date creationTime) {
+        this.creationTime = creationTime;
+    }
+    
     @Override
     public boolean equals(Object object) {
         // TODO: Warning - this method won't work in the case the id fields are not set
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index fd309790026..36a4cff6882 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -179,10 +179,10 @@ private void initMaps() {
 
                 // populate the file access requests map
                 for (FileAccessRequest fileAccessRequest : file.getFileAccessRequests()) {
-                    List<FileAccessRequest> requestedFiles = fileAccessRequestMap.get(fileAccessRequest.getAuthenticatedUser());
+                    List<FileAccessRequest> requestedFiles = fileAccessRequestMap.get(fileAccessRequest.getRequester());
                     if (requestedFiles == null) {
                         requestedFiles = new ArrayList<>();
-                        AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getAuthenticatedUser().getUserIdentifier());
+                        AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getRequester().getUserIdentifier());
                         fileAccessRequestMap.put(withProvider, requestedFiles);
                     }
                     requestedFiles.add(fileAccessRequest);
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 02441a9ee11..358b0159e2a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1489,7 +1489,7 @@ public Response listFileAccessRequests(@Context ContainerRequestContext crc, @Pa
         JsonArrayBuilder userArray = Json.createArrayBuilder();
 
         for (FileAccessRequest fileAccessRequest : requests) {
-            userArray.add(json(fileAccessRequest.getAuthenticatedUser()));
+            userArray.add(json(fileAccessRequest.getRequester()));
         }
 
         return ok(userArray);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index f6a3b287778..db0054e375e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -52,38 +52,20 @@ public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean s
         this.fileAccessRequest = new FileAccessRequest(file,requester);
         this.sendNotification = sendNotification;
     }
-        
-        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) {
-            // for data file check permission on owning dataset
-            super(dvRequest, file);        
-            this.file = file;        
-            this.requester = (AuthenticatedUser) dvRequest.getUser();
-            this.fileAccessRequest = new FileAccessRequest(file,requester,gbr);
-            this.sendNotification = false;
-        }
 
-        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, Boolean sendNotification) {
-            // for data file check permission on owning dataset
-            super(dvRequest, file);        
-            this.file = file;
-            this.requester = (AuthenticatedUser) dvRequest.getUser();
-            this.fileAccessRequest = new FileAccessRequest(file,requester,gbr);
-            this.sendNotification = sendNotification;
-        }
+    public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) {
+        this(dvRequest, file, gbr, false);
+    }
 
-        
-        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr) {
-            this(dvRequest, file, gbr, false);
-        }
+    public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, Boolean sendNotification) {
+        // for data file check permission on owning dataset
+        super(dvRequest, file);
+        this.file = file;
+        this.requester = (AuthenticatedUser) dvRequest.getUser();
+        this.fileAccessRequest = new FileAccessRequest(file, requester, gbr);
+        this.sendNotification = sendNotification;
+    }
 
-        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, Boolean sendNotification) {
-            // for data file check permission on owning dataset
-            super(dvRequest, file);        
-            this.file = file;
-            this.requester = (AuthenticatedUser) dvRequest.getUser();
-            this.fileAccessRequest = new FileAccessRequest(file,requester,gbr);
-            this.sendNotification = sendNotification;
-        }
     @Override
     public DataFile execute(CommandContext ctxt) throws CommandException {
 

From 10f8a0a1470fd62e75b84defc0ad752de6175658 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 19 May 2023 12:36:24 -0400
Subject: [PATCH 089/922] update script naming and content

---
 ....sql => V5.13.0.1__8092-timestamp-of-data-access-request.sql} | 0
 ...stbook-on-request.sql => V5.13.0.2__guestbook-on-request.sql} | 1 +
 2 files changed, 1 insertion(+)
 rename src/main/resources/db/migration/{V5.13.0.4__8092-timestamp-of-data-access-request.sql => V5.13.0.1__8092-timestamp-of-data-access-request.sql} (100%)
 rename src/main/resources/db/migration/{V5.13.0.3__guestbook-on-request.sql => V5.13.0.2__guestbook-on-request.sql} (85%)

diff --git a/src/main/resources/db/migration/V5.13.0.4__8092-timestamp-of-data-access-request.sql b/src/main/resources/db/migration/V5.13.0.1__8092-timestamp-of-data-access-request.sql
similarity index 100%
rename from src/main/resources/db/migration/V5.13.0.4__8092-timestamp-of-data-access-request.sql
rename to src/main/resources/db/migration/V5.13.0.1__8092-timestamp-of-data-access-request.sql
diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.2__guestbook-on-request.sql
similarity index 85%
rename from src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
rename to src/main/resources/db/migration/V5.13.0.2__guestbook-on-request.sql
index f28e7ca8896..20bc1fc1c76 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.2__guestbook-on-request.sql
@@ -3,4 +3,5 @@ ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT;
+ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_guestbookresponse;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);

From 560faf582007106089caf13ab9970498e057ae2e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 19 May 2023 13:12:15 -0400
Subject: [PATCH 090/922] remove second action (merge issue)

---
 src/main/webapp/filesFragment.xhtml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index fa49936591c..94ee2fc8b43 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -481,8 +481,7 @@
                 </div>
 
                 <p:commandLink rendered="#{DatasetPage.fileAccessRequestMultiButtonRequired}"
-                               styleClass="btn btn-default btn-request"                                   
-                               action="#{DatasetPage.requestAccessMultipleFiles()}"
+                               styleClass="btn btn-default btn-request"
                                update="@form, @([id$=messagePanel])" 
                                action="#{DatasetPage.validateFilesForRequestAccess()}"
                                disabled="#{DatasetPage.locked or !DatasetPage.fileAccessRequestMultiButtonEnabled}">

From bc42df0946371c1af41560b79b5324ed5b565b99 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 19 May 2023 13:12:26 -0400
Subject: [PATCH 091/922] typo from merge

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index c0b4fc104ad..a3160a6e48b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3127,7 +3127,7 @@ public void startDownloadSelectedOriginal() {
 
     private void startDownload(boolean downloadOriginal){
         boolean guestbookRequired = isDownloadPopupRequired();
-        boolean validate = validateFilesForDownload(guestbookRequired);
+        boolean validate = validateFilesForDownload(downloadOriginal);
         if (validate) {
             updateGuestbookResponse(guestbookRequired, downloadOriginal);
             if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){

From 57e984b0e468a55f578b3b21e4787c1a63e4dca9 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 19 May 2023 13:27:52 -0400
Subject: [PATCH 092/922] fix for #9601

---
 .../edu/harvard/iq/dataverse/ingest/IngestServiceBean.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 7cdfda8d082..5a353453fe8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -332,9 +332,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 				} catch (IOException e) {
 					logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
 				} 
-				if (unattached) {
-					dataFile.setOwner(null);
-				}
+
 				if (savedSuccess && belowLimit) {
 					// These are all brand new files, so they should all have
 					// one filemetadata total. -- L.A.
@@ -388,6 +386,9 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                         dataFile.setContentType(FileUtil.MIME_TYPE_TSV);
                     }
 				}
+                if (unattached) {
+                    dataFile.setOwner(null);
+                }
 				// ... and let's delete the main temp file if it exists:
 				if(tempLocationPath!=null) {
     				try {

From 0c76f7b02b4214efa37b5ac7a5d23f308afae5d5 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 19 May 2023 15:53:09 -0400
Subject: [PATCH 093/922] remove QDR updates

---
 src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java
index 34176f7fb26..1d481f18cf5 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/FileUtilTest.java
@@ -138,7 +138,7 @@ public void testIsDownloadPopupRequiredLicenseCC0() {
             DatasetVersion dsv1 = new DatasetVersion();
             dsv1.setVersionState(DatasetVersion.VersionState.RELEASED);
             TermsOfUseAndAccess termsOfUseAndAccess = new TermsOfUseAndAccess();
-            License license = new License("CC0 1.0", "Creative Commons CC0 1.0 Universal Public Domain Dedication.", URI.create("https://creativecommons.org/publicdomain/zero/1.0"), URI.create("https://licensebuttons.net/p/zero/1.0/88x31.png"), true,1L);
+            License license = new License("CC0", "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), true, 1l);
             license.setDefault(true);
             termsOfUseAndAccess.setLicense(license);
             dsv1.setTermsOfUseAndAccess(termsOfUseAndAccess);

From 45d9042a226cc53cb8f8a09902a849889786cc0b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Fri, 19 May 2023 23:42:32 +0200
Subject: [PATCH 094/922] style(api): fix typos and style issues in
 BearerTokenAuthMechanism

---
 .../api/auth/BearerTokenAuthMechanism.java          | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
index c4b03728179..856670523b1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
@@ -23,6 +23,8 @@
 
 public class BearerTokenAuthMechanism implements AuthMechanism {
     private static final String BEARER_AUTH_SCHEME = "Bearer";
+    private static final Logger logger = Logger.getLogger(BearerTokenAuthMechanism.class.getCanonicalName());
+    
     public static final String UNAUTHORIZED_BEARER_TOKEN = "Unauthorized bearer token";
     public static final String INVALID_BEARER_TOKEN = "Could not parse bearer token";
     public static final String BEARER_TOKEN_DETECTED_NO_OIDC_PROVIDER_CONFIGURED = "Bearer token detected, no OIDC provider configured";
@@ -31,18 +33,19 @@ public class BearerTokenAuthMechanism implements AuthMechanism {
     protected AuthenticationServiceBean authSvc;
     @Inject
     protected UserServiceBean userSvc;
-    private static final Logger logger = Logger.getLogger(BearerTokenAuthMechanism.class.getCanonicalName());
+    
     @Override
     public User findUserFromRequest(ContainerRequestContext containerRequestContext) throws WrappedAuthErrorResponse {
         if (FeatureFlags.API_BEARER_AUTH.enabled()) {
             Optional<String> bearerToken = getRequestApiKey(containerRequestContext);
             // No Bearer Token present, hence no user can be authenticated
-            if (!bearerToken.isPresent()) {
+            if (bearerToken.isEmpty()) {
                 return null;
             }
+            
             // Validate and verify provided Bearer Token, and retrieve UserRecordIdentifier
             // TODO: Get the identifier from an invalidating cache to avoid lookup bursts of the same token. Tokens in the cache should be removed after some (configurable) time.
-            UserRecordIdentifier userInfo = verifyOidcBearerTokenAndGetUserIndentifier(bearerToken.get());
+            UserRecordIdentifier userInfo = verifyOidcBearerTokenAndGetUserIdentifier(bearerToken.get());
 
             // retrieve Authenticated User from AuthService
             AuthenticatedUser authUser = authSvc.lookupUser(userInfo);
@@ -67,7 +70,7 @@ public User findUserFromRequest(ContainerRequestContext containerRequestContext)
      * @param token The string containing the encoded JWT
      * @return
      */
-    private UserRecordIdentifier verifyOidcBearerTokenAndGetUserIndentifier(String token) throws WrappedAuthErrorResponse {
+    private UserRecordIdentifier verifyOidcBearerTokenAndGetUserIdentifier(String token) throws WrappedAuthErrorResponse {
         try {
             BearerAccessToken accessToken = BearerAccessToken.parse(token);
             // Get list of all authentication providers using Open ID Connect
@@ -108,7 +111,7 @@ private UserRecordIdentifier verifyOidcBearerTokenAndGetUserIndentifier(String t
      * Retrieve the raw, encoded token value from the Authorization Bearer HTTP header as defined in RFC 6750
      * @return An {@link Optional} either empty if not present or the raw token from the header
      */
-    private  Optional<String> getRequestApiKey(ContainerRequestContext containerRequestContext) {
+    private Optional<String> getRequestApiKey(ContainerRequestContext containerRequestContext) {
         String headerParamApiKey = containerRequestContext.getHeaderString(HttpHeaders.AUTHORIZATION);
         if (headerParamApiKey != null && headerParamApiKey.toLowerCase().startsWith(BEARER_AUTH_SCHEME.toLowerCase() + " ")) {
             return Optional.of(headerParamApiKey);

From bb49ea52f482b1b6466e124ca926453670699b09 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Fri, 19 May 2023 23:47:31 +0200
Subject: [PATCH 095/922] refactor(api,auth): improve
 OIDCAuthProvider.getUserIdentifier

- Reuse more existing code paths to avoid duplication
- Make actual use of returning an empty optional
- Remove no longer checked exception OAuth2Exception
- Improve Javadocs of method
- Don't just retrieve and bail out on fail but provide smaller analysis steps with logs
- Rename method to be more concise in name selection
- Change BearerTokenAuthMechanism accordingly
---
 .../api/auth/BearerTokenAuthMechanism.java    |  5 +-
 .../oauth2/oidc/OIDCAuthProvider.java         | 47 +++++++++++++++----
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
index 856670523b1..eeabcba9f06 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
@@ -5,7 +5,6 @@
 import edu.harvard.iq.dataverse.UserServiceBean;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
-import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
@@ -88,12 +87,12 @@ private UserRecordIdentifier verifyOidcBearerTokenAndGetUserIdentifier(String to
             for (OIDCAuthProvider provider : providers) {
                 try {
                     // The OIDCAuthProvider need to verify a Bearer Token and equip the client means to identify the corresponding AuthenticatedUser.
-                    Optional<UserRecordIdentifier> userInfo = provider.getUserIdentifierForValidToken(accessToken);
+                    Optional<UserRecordIdentifier> userInfo = provider.getUserIdentifier(accessToken);
                     if(userInfo.isPresent()) {
                         logger.log(Level.FINE, "Bearer token detected, provider {0} confirmed validity and provided identifier", provider.getId());
                         return userInfo.get();
                     }
-                } catch ( IOException| OAuth2Exception e) {
+                } catch (IOException e) {
                     logger.log(Level.FINE, "Bearer token detected, provider " + provider.getId() + " indicates an invalid Token, skipping", e);
                 }
             }
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
index 4cf2eeb626a..52362f7abeb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
@@ -46,6 +46,7 @@
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
@@ -305,16 +306,42 @@ Optional<UserInfo> getUserInfo(BearerAccessToken accessToken) throws IOException
     }
 
     /**
-     * Returns the UserRecordIdentifier corresponding to the given accessToken if valid.
-     * UserRecordIdentifier (same used as in OAuth2UserRecord), i.e. can be used to find a local UserAccount.
-     * @param accessToken
-     * @return Returns the UserRecordIdentifier corresponding to the given accessToken if valid.
-     * @throws IOException
-     * @throws OAuth2Exception
+     * Trades an access token for an {@link UserRecordIdentifier} (if valid).
+     *
+     * @apiNote The resulting {@link UserRecordIdentifier} may be used with
+     *          {@link edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean#lookupUser(UserRecordIdentifier)}
+     *          to look up an {@link edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser} from the database.
+     * @see edu.harvard.iq.dataverse.api.auth.BearerTokenAuthMechanism
+     *
+     * @param accessToken The token to use when requesting user information from the provider
+     * @return Returns an {@link UserRecordIdentifier} for a valid access token or an empty {@link Optional}.
+     * @throws IOException In case communication with the endpoint fails to succeed for an I/O reason
      */
-    public Optional<UserRecordIdentifier> getUserIdentifierForValidToken(BearerAccessToken accessToken) throws IOException, OAuth2Exception{
-        // Request the UserInfoEndpoint to obtain UserInfo, since this endpoint also validate the Token we can reuse the existing code path.
-        // As an alternative we could use the Introspect Endpoint or assume the Token as some encoded information (i.e. JWT).
-        return  Optional.of(new UserRecordIdentifier(  this.getId(), getUserInfo(accessToken).get().getSubject().getValue()));
+    public Optional<UserRecordIdentifier> getUserIdentifier(BearerAccessToken accessToken) throws IOException {
+        OAuth2UserRecord userRecord;
+        try {
+            // Try to retrieve with given token (throws if invalid token)
+            Optional<UserInfo> userInfo = getUserInfo(accessToken);
+            
+            if (userInfo.isPresent()) {
+                // Take this detour to avoid code duplication and potentially hard to track conversion errors.
+                userRecord = getUserRecord(userInfo.get());
+            } else {
+                // This should not happen - an error at the provider side will lead to an exception.
+                logger.log(Level.WARNING,
+                    "User info retrieval from {0} returned empty optional but expected exception for token {1}.",
+                    List.of(getId(), accessToken).toArray()
+                );
+                return Optional.empty();
+            }
+        } catch (OAuth2Exception e) {
+            logger.log(Level.FINE,
+                "Could not retrieve user info with token {0} at provider {1}: {2}",
+                List.of(accessToken, getId(), e.getMessage()).toArray());
+            logger.log(Level.FINER, "Retrieval failed, details as follows: ", e);
+            return Optional.empty();
+        }
+        
+        return Optional.of(userRecord.getUserRecordIdentifier());
     }
 }

From cecb034ad7e2be7c47496e0197a7c591e2be503e Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Sat, 20 May 2023 00:01:38 +0200
Subject: [PATCH 096/922] refactor(api): add TODO in BearerTokenAuthMechanism
 not to neglect IO exceptions

---
 .../iq/dataverse/api/auth/BearerTokenAuthMechanism.java        | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
index eeabcba9f06..e26717e97b1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanism.java
@@ -93,6 +93,9 @@ private UserRecordIdentifier verifyOidcBearerTokenAndGetUserIdentifier(String to
                         return userInfo.get();
                     }
                 } catch (IOException e) {
+                    // TODO: Just logging this is not sufficient - if there is an IO error with the one provider
+                    //       which would have validated successfully, this is not the users fault. We need to
+                    //       take note and refer to that later when occurred.
                     logger.log(Level.FINE, "Bearer token detected, provider " + provider.getId() + " indicates an invalid Token, skipping", e);
                 }
             }

From 647315cb52fbdbb45fb33492bdcbd9c829c73a16 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Sun, 21 May 2023 22:49:47 +0200
Subject: [PATCH 097/922] feat(test): extend JUnit5 test helper JvmSettings

- Adding a @JvmSetting to a testclass now applies it before all test methods (will be reset if necessary after all tests ran)
- Enable deleting existing settings (which allows to override a class wide setting for example) by setting value to "null"
- Introduce settings broker with a first simple local implementation as extension point to set these settings out somewhere (TM).
- Broker requires annotation which one to use within a class at class level
---
 .../util/testing/JvmSettingBroker.java        |  43 +++++
 .../util/testing/JvmSettingExtension.java     | 164 ++++++++++++------
 .../util/testing/LocalJvmSettings.java        |  39 +++++
 3 files changed, 191 insertions(+), 55 deletions(-)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingBroker.java
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/util/testing/LocalJvmSettings.java

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingBroker.java b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingBroker.java
new file mode 100644
index 00000000000..1235df89b3e
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingBroker.java
@@ -0,0 +1,43 @@
+package edu.harvard.iq.dataverse.util.testing;
+
+import java.io.IOException;
+
+/**
+ * Provide an interface to access and manipulate {@link edu.harvard.iq.dataverse.settings.JvmSettings}
+ * at some place (local, remote, different ways to access, etc.).
+ * Part of the {@link JvmSettingExtension} extension to allow JUnit5 tests to manipulate these
+ * settings, enabling to test different code paths and so on.
+ * @implNote Keep in mind to use methods that do not require restarts or similar to set or delete a setting.
+ *           This must be changeable on the fly, otherwise it will be useless for testing.
+ *           Yes, non-hot-reloadable settings may be a problem. The code should be refactored in these cases.
+ */
+public interface JvmSettingBroker {
+    
+    /**
+     * Receive the value of a {@link edu.harvard.iq.dataverse.settings.JvmSettings} given as its {@link String}
+     * representation. The reason for this is that we may have inserted variable names already.
+     * @param key The JVM setting to receive as key, e.g. "dataverse.fqdn".
+     * @return The value of the setting if present or null.
+     * @throws IOException When communication goes sideways.
+     */
+    String getJvmSetting(String key) throws IOException;
+    
+    /**
+     * Set the value of a {@link edu.harvard.iq.dataverse.settings.JvmSettings} (given as its {@link String}
+     * representation). The reason for this is that we may have inserted variable names already.
+     * @param key The JVM setting to receive as key, e.g. "dataverse.fqdn".
+     * @param value The JVM setting's value we want to have it set to.
+     * @throws IOException When communication goes sideways.
+     */
+    void setJvmSetting(String key, String value) throws IOException;
+    
+    /**
+     * Remove the value of a {@link edu.harvard.iq.dataverse.settings.JvmSettings} (given as its {@link String}
+     * representation). For some tests, one might want to clear a certain setting again and potentially have it set
+     * back afterward. The reason for this is that we may have inserted variable names already.
+     * @param key The JVM setting to receive as key, e.g. "dataverse.fqdn".
+     * @throws IOException When communication goes sideways.
+     */
+    String deleteJvmSetting(String key) throws IOException;
+    
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java
index 17728e75ffc..2065d7b3ae6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/testing/JvmSettingExtension.java
@@ -1,81 +1,124 @@
 package edu.harvard.iq.dataverse.util.testing;
 
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import org.junit.jupiter.api.extension.AfterAllCallback;
 import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
 import org.junit.jupiter.api.extension.BeforeTestExecutionCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.platform.commons.support.AnnotationSupport;
+import org.junit.platform.commons.support.ReflectionSupport;
 
-import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.List;
+import java.util.Optional;
 
 import static edu.harvard.iq.dataverse.util.testing.JvmSetting.PLACEHOLDER;
 
-public class JvmSettingExtension implements BeforeTestExecutionCallback, AfterTestExecutionCallback {
+public class JvmSettingExtension implements BeforeTestExecutionCallback, AfterTestExecutionCallback, BeforeAllCallback, AfterAllCallback {
     
-    private ExtensionContext.Store getStore(ExtensionContext context) {
-        return context.getStore(ExtensionContext.Namespace.create(getClass(), context.getRequiredTestClass(), context.getRequiredTestMethod()));
+    @Override
+    public void beforeAll(ExtensionContext extensionContext) throws Exception {
+        List<JvmSetting> settings = AnnotationSupport.findRepeatableAnnotations(extensionContext.getTestClass(), JvmSetting.class);
+        ExtensionContext.Store store = extensionContext.getStore(
+            ExtensionContext.Namespace.create(getClass(), extensionContext.getRequiredTestClass()));
+        
+        setSetting(extensionContext.getRequiredTestClass(), settings, getBroker(extensionContext), store);
+    }
+    
+    @Override
+    public void afterAll(ExtensionContext extensionContext) throws Exception {
+        List<JvmSetting> settings = AnnotationSupport.findRepeatableAnnotations(extensionContext.getTestClass(), JvmSetting.class);
+        ExtensionContext.Store store = extensionContext.getStore(
+            ExtensionContext.Namespace.create(getClass(), extensionContext.getRequiredTestClass()));
+        
+        resetSetting(settings, getBroker(extensionContext), store);
     }
     
     @Override
     public void beforeTestExecution(ExtensionContext extensionContext) throws Exception {
-        extensionContext.getTestMethod().ifPresent(method -> {
-            JvmSetting[] settings = method.getAnnotationsByType(JvmSetting.class);
-            for (JvmSetting setting : settings) {
-                // get the setting name (might need var args substitution)
-                String settingName = getSettingName(setting);
-                
-                // get the setting ...
-                String oldSetting = System.getProperty(settingName);
+        List<JvmSetting> settings = AnnotationSupport.findRepeatableAnnotations(extensionContext.getTestMethod(), JvmSetting.class);
+        ExtensionContext.Store store = extensionContext.getStore(
+            ExtensionContext.Namespace.create(
+                getClass(),
+                extensionContext.getRequiredTestClass(),
+                extensionContext.getRequiredTestMethod()
+            ));
+        
+        setSetting(extensionContext.getRequiredTestClass(), settings, getBroker(extensionContext), store);
+    }
     
-                // if present - store in context to restore later
-                if (oldSetting != null) {
-                    getStore(extensionContext).put(settingName, oldSetting);
-                }
-                
-                // set to new value
-                if (setting.value().equals(PLACEHOLDER) && setting.method().equals(PLACEHOLDER)) {
-                    throw new IllegalArgumentException("You must either provide a value or a method reference " +
-                        "for key JvmSettings." + setting.key());
-                }
-                
-                // retrieve value from static test class method if no setting given
-                if (setting.value().equals(PLACEHOLDER)) {
-                    extensionContext.getTestClass().ifPresent(klass -> {
-                        try {
-                            Method valueMethod = klass.getDeclaredMethod(setting.method());
-                            valueMethod.setAccessible(true);
-                            System.setProperty(settingName, (String)valueMethod.invoke(null));
-                        } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
-                            throw new RuntimeException(e);
-                        }
-                    });
-                } else {
-                    System.setProperty(settingName, setting.value());
+    @Override
+    public void afterTestExecution(ExtensionContext extensionContext) throws Exception {
+        List<JvmSetting> settings = AnnotationSupport.findRepeatableAnnotations(extensionContext.getTestMethod(), JvmSetting.class);
+        ExtensionContext.Store store = extensionContext.getStore(
+            ExtensionContext.Namespace.create(
+                getClass(),
+                extensionContext.getRequiredTestClass(),
+                extensionContext.getRequiredTestMethod()
+            ));
+        
+        resetSetting(settings, getBroker(extensionContext), store);
+    }
+    
+    private void setSetting(Class<?> testClass, List<JvmSetting> settings, JvmSettingBroker broker, ExtensionContext.Store store) throws Exception {
+        for (JvmSetting setting : settings) {
+            // get the setting name (might need var args substitution)
+            String settingName = getSettingName(setting);
+            
+            // get the setting value ...
+            String oldSetting = broker.getJvmSetting(settingName);
+            
+            // if present - store in context to restore later
+            if (oldSetting != null) {
+                store.put(settingName, oldSetting);
+            }
+            
+            // set to new value
+            if (setting.value().equals(PLACEHOLDER) && setting.method().equals(PLACEHOLDER)) {
+                throw new IllegalArgumentException("You must either provide a value or a method reference " +
+                    "for key JvmSettings" + setting.key());
+            }
+            
+            String value;
+            // Retrieve value from static (!) test class method if no direct setting given
+            if (setting.value().equals(PLACEHOLDER)) {
+                Optional<Method> valueMethod = ReflectionSupport.findMethod(testClass, setting.method());
+                if (valueMethod.isEmpty() || ! Modifier.isStatic(valueMethod.get().getModifiers())) {
+                    throw new IllegalStateException("Could not find a static method '" + setting.method() + "' in test class");
                 }
+                value = (String) ReflectionSupport.invokeMethod(valueMethod.get(), null);
+            // Set to new value by using the directly given value
+            } else {
+                value = setting.value();
             }
-        });
+            
+            // If the retrieved value is null, delete the setting (will be reset after the test), otherwise set.
+            if (value != null) {
+                broker.setJvmSetting(settingName, value);
+            } else if (oldSetting != null) {
+                broker.deleteJvmSetting(settingName);
+            }
+        }
     }
     
-    @Override
-    public void afterTestExecution(ExtensionContext extensionContext) throws Exception {
-        extensionContext.getTestMethod().ifPresent(method -> {
-            JvmSetting[] settings = method.getAnnotationsByType(JvmSetting.class);
-            for (JvmSetting setting : settings) {
-                // get the setting name (might need var args substitution)
-                String settingName = getSettingName(setting);
-                
-                // get a stored setting from context
-                String oldSetting = getStore(extensionContext).remove(settingName, String.class);
-                
-                // if present before, restore
-                if (oldSetting != null) {
-                    System.setProperty(settingName, oldSetting);
+    private void resetSetting(List<JvmSetting> settings, JvmSettingBroker broker, ExtensionContext.Store store) throws Exception {
+        for (JvmSetting setting : settings) {
+            // get the setting name (might need var args substitution)
+            String settingName = getSettingName(setting);
+            
+            // get a stored setting from context
+            String oldSetting = store.remove(settingName, String.class);
+            
+            // if present before, restore
+            if (oldSetting != null) {
+                broker.setJvmSetting(settingName, oldSetting);
                 // if NOT present before, delete
-                } else {
-                    System.clearProperty(settingName);
-                }
+            } else {
+                broker.deleteJvmSetting(settingName);
             }
-        });
+        }
     }
     
     private String getSettingName(JvmSetting setting) {
@@ -95,4 +138,15 @@ private String getSettingName(JvmSetting setting) {
         
         return target.getScopedKey();
     }
+    
+    private JvmSettingBroker getBroker(ExtensionContext extensionContext) throws Exception {
+        // Is this test class using local system properties, then get a broker for these
+        if (AnnotationSupport.isAnnotated(extensionContext.getTestClass(), LocalJvmSettings.class)) {
+            return LocalJvmSettings.localBroker;
+        // NOTE: this might be extended later with other annotations to support other means of handling the settings
+        } else {
+            throw new IllegalStateException("You must provide the @LocalJvmSettings annotation to the test class");
+        }
+    }
+    
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/testing/LocalJvmSettings.java b/src/test/java/edu/harvard/iq/dataverse/util/testing/LocalJvmSettings.java
new file mode 100644
index 00000000000..372fa91f6f6
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/util/testing/LocalJvmSettings.java
@@ -0,0 +1,39 @@
+package edu.harvard.iq.dataverse.util.testing;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * This annotation expresses that a test class wants to manipulate local
+ * settings (because the tests run within the same JVM as the code itself).
+ * This is mostly true for unit tests.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+@ExtendWith(JvmSettingExtension.class)
+@Inherited
+public @interface LocalJvmSettings {
+    
+    JvmSettingBroker localBroker = new JvmSettingBroker() {
+        @Override
+        public String getJvmSetting(String key) {
+            return System.getProperty(key);
+        }
+        
+        @Override
+        public void setJvmSetting(String key, String value) {
+            System.setProperty(key, value);
+        }
+        
+        @Override
+        public String deleteJvmSetting(String key) {
+            return System.clearProperty(key);
+        }
+    };
+    
+}
\ No newline at end of file

From 645770f0c08c042f934707d88c187de34ebab95b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Sun, 21 May 2023 22:59:20 +0200
Subject: [PATCH 098/922] refactor(test): make existing test using JvmSetting
 annotated with @LocalJvmSettings

---
 .../dataverse/api/auth/BearerTokenAuthMechanismTest.java  | 4 +++-
 .../api/auth/SessionCookieAuthMechanismTest.java          | 8 +++++---
 .../iq/dataverse/export/SchemaDotOrgExporterTest.java     | 2 ++
 .../dataverse/externaltools/ExternalToolHandlerTest.java  | 2 ++
 .../harvard/iq/dataverse/search/IndexServiceBeanTest.java | 2 ++
 .../iq/dataverse/search/SolrClientServiceTest.java        | 2 ++
 .../harvard/iq/dataverse/settings/FeatureFlagsTest.java   | 2 ++
 .../harvard/iq/dataverse/settings/JvmSettingsTest.java    | 2 ++
 .../edu/harvard/iq/dataverse/util/SystemConfigTest.java   | 2 ++
 .../edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java   | 2 ++
 10 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
index 0370daa5ea2..b38300df660 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
@@ -12,6 +12,7 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
@@ -25,7 +26,8 @@
 import static edu.harvard.iq.dataverse.api.auth.BearerTokenAuthMechanism.*;
 import static org.junit.jupiter.api.Assertions.*;
 
-public class BearerTokenAuthMechanismTest {
+@LocalJvmSettings
+class BearerTokenAuthMechanismTest {
 
     private static final String TEST_API_KEY = "test-api-key";
 
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/SessionCookieAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/SessionCookieAuthMechanismTest.java
index 477f8ee377a..74a7d239c05 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/auth/SessionCookieAuthMechanismTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/SessionCookieAuthMechanismTest.java
@@ -6,6 +6,7 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
@@ -13,7 +14,8 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
-public class SessionCookieAuthMechanismTest {
+@LocalJvmSettings
+class SessionCookieAuthMechanismTest {
 
     private SessionCookieAuthMechanism sut;
 
@@ -24,7 +26,7 @@ public void setUp() {
 
     @Test
     @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "false", varArgs = "api-session-auth")
-    public void testFindUserFromRequest_FeatureFlagDisabled() throws WrappedAuthErrorResponse {
+    void testFindUserFromRequest_FeatureFlagDisabled() throws WrappedAuthErrorResponse {
         sut.session = Mockito.mock(DataverseSession.class);
 
         User actual = sut.findUserFromRequest(new ContainerRequestTestFake());
@@ -34,7 +36,7 @@ public void testFindUserFromRequest_FeatureFlagDisabled() throws WrappedAuthErro
 
     @Test
     @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-session-auth")
-    public void testFindUserFromRequest_FeatureFlagEnabled_UserAuthenticated() throws WrappedAuthErrorResponse {
+    void testFindUserFromRequest_FeatureFlagEnabled_UserAuthenticated() throws WrappedAuthErrorResponse {
         DataverseSession dataverseSessionStub = Mockito.mock(DataverseSession.class);
         User testAuthenticatedUser = new AuthenticatedUser();
         Mockito.when(dataverseSessionStub.getUser()).thenReturn(testAuthenticatedUser);
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java
index e660cf78da2..722b74406d4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/export/SchemaDotOrgExporterTest.java
@@ -33,6 +33,7 @@
 import javax.json.JsonObject;
 
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
@@ -44,6 +45,7 @@
 /**
  * For docs see {@link SchemaDotOrgExporter}.
  */
+@LocalJvmSettings
 public class SchemaDotOrgExporterTest {
 
     private static final Logger logger = Logger.getLogger(SchemaDotOrgExporterTest.class.getCanonicalName());
diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
index ab3a0263d66..c77d59123e4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
@@ -10,6 +10,7 @@
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.Test;
 
 import javax.json.Json;
@@ -21,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
+@LocalJvmSettings
 public class ExternalToolHandlerTest {
 
     // TODO: It would probably be better to split these into individual tests.
diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
index dd3dc2c6c95..ce6005a3d11 100644
--- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
@@ -18,6 +18,7 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.junit.jupiter.api.BeforeEach;
@@ -37,6 +38,7 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+@LocalJvmSettings
 @ExtendWith(MockitoExtension.class)
 public class IndexServiceBeanTest {
     private static final Logger logger = Logger.getLogger(IndexServiceBeanTest.class.getCanonicalName());
diff --git a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java
index a3b3c8a2080..72eafcd763c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/search/SolrClientServiceTest.java
@@ -4,6 +4,7 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -14,6 +15,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+@LocalJvmSettings
 @ExtendWith(MockitoExtension.class)
 class SolrClientServiceTest {
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/FeatureFlagsTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/FeatureFlagsTest.java
index 1a9fdeaa3da..26f2186695d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/settings/FeatureFlagsTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/settings/FeatureFlagsTest.java
@@ -1,10 +1,12 @@
 package edu.harvard.iq.dataverse.settings;
 
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.*;
 
+@LocalJvmSettings
 class FeatureFlagsTest {
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/JvmSettingsTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/JvmSettingsTest.java
index 559d00fe0b7..6b03f20fc41 100644
--- a/src/test/java/edu/harvard/iq/dataverse/settings/JvmSettingsTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/settings/JvmSettingsTest.java
@@ -1,11 +1,13 @@
 package edu.harvard.iq.dataverse.settings;
 
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import edu.harvard.iq.dataverse.util.testing.SystemProperty;
 import org.junit.jupiter.api.Test;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+@LocalJvmSettings
 class JvmSettingsTest {
     @Test
     @JvmSetting(key = JvmSettings.VERSION, value = "foobar")
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java
index 2806aa3aa9b..82b89bca678 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java
@@ -3,6 +3,7 @@
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -15,6 +16,7 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.doReturn;
 
+@LocalJvmSettings
 @ExtendWith(MockitoExtension.class)
 class SystemConfigTest {
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java
index 8310af8885c..d70a108e7c6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/UrlTokenUtilTest.java
@@ -9,6 +9,7 @@
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
@@ -16,6 +17,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+@LocalJvmSettings
 class UrlTokenUtilTest {
 
     @Test

From dd80162cb7f1b6ad5268057cdb7da547c9bf4b62 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Sun, 21 May 2023 23:01:15 +0200
Subject: [PATCH 099/922] style(test,api,auth): make
 BearerTokenAuthMechanismTest simpler

---
 .../auth/BearerTokenAuthMechanismTest.java    | 55 +++++--------------
 1 file changed, 15 insertions(+), 40 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
index b38300df660..8a57ee4c41c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
@@ -27,6 +27,7 @@
 import static org.junit.jupiter.api.Assertions.*;
 
 @LocalJvmSettings
+@JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
 class BearerTokenAuthMechanismTest {
 
     private static final String TEST_API_KEY = "test-api-key";
@@ -36,14 +37,12 @@ class BearerTokenAuthMechanismTest {
     @BeforeEach
     public void setUp() {
         sut = new BearerTokenAuthMechanism();
-    }
-
-    @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_no_token() throws WrappedAuthErrorResponse {
         sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
         sut.userSvc = Mockito.mock(UserServiceBean.class);
+    }
 
+    @Test
+    void testFindUserFromRequest_no_token() throws WrappedAuthErrorResponse {
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake(null);
         User actual = sut.findUserFromRequest(testContainerRequest);
 
@@ -51,11 +50,9 @@ public void testFindUserFromRequest_no_token() throws WrappedAuthErrorResponse {
     }
 
     @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_invalid_token() throws WrappedAuthErrorResponse {
-        sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
-        sut.userSvc = Mockito.mock(UserServiceBean.class);
+    void testFindUserFromRequest_invalid_token() throws WrappedAuthErrorResponse {
         Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.emptySet());
+        
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer ");
         WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest));
 
@@ -63,11 +60,9 @@ public void testFindUserFromRequest_invalid_token() throws WrappedAuthErrorRespo
         assertEquals(INVALID_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage());
     }
     @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_no_OidcProvider() throws WrappedAuthErrorResponse {
-        sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
-        sut.userSvc = Mockito.mock(UserServiceBean.class);
+    void testFindUserFromRequest_no_OidcProvider() throws WrappedAuthErrorResponse {
         Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.emptySet());
+        
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " +TEST_API_KEY);
         WrappedAuthErrorResponse wrappedAuthErrorResponse = assertThrows(WrappedAuthErrorResponse.class, () -> sut.findUserFromRequest(testContainerRequest));
 
@@ -76,12 +71,7 @@ public void testFindUserFromRequest_no_OidcProvider() throws WrappedAuthErrorRes
     }
 
     @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_oneProvider_invalidToken_1() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
-
-        sut.userSvc = Mockito.mock(UserServiceBean.class);
-        sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
-
+    void testFindUserFromRequest_oneProvider_invalidToken_1() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -91,7 +81,7 @@ public void testFindUserFromRequest_oneProvider_invalidToken_1() throws WrappedA
 
         // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token
         BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY);
-        Mockito.when(oidcAuthProvider.getUserIdentifierForValidToken(token)).thenReturn(Optional.empty());
+        Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenReturn(Optional.empty());
 
         // when
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " + TEST_API_KEY);
@@ -102,12 +92,7 @@ public void testFindUserFromRequest_oneProvider_invalidToken_1() throws WrappedA
     }
 
     @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_oneProvider_invalidToken_2() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
-
-        sut.userSvc = Mockito.mock(UserServiceBean.class);
-        sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
-
+    void testFindUserFromRequest_oneProvider_invalidToken_2() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -127,12 +112,7 @@ public void testFindUserFromRequest_oneProvider_invalidToken_2() throws WrappedA
         assertEquals(UNAUTHORIZED_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage());
     }
     @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
-
-        sut.userSvc = Mockito.mock(UserServiceBean.class);
-        sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
-
+    void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -143,7 +123,7 @@ public void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthE
         // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token
         UserRecordIdentifier userinfo = new UserRecordIdentifier(providerID, "KEY");
         BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY);
-        Mockito.when(oidcAuthProvider.getUserIdentifierForValidToken(token)).thenReturn(Optional.of(userinfo));
+        Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenReturn(Optional.of(userinfo));
 
         // ensures that the AuthenticationServiceBean can retrieve an Authenticated user based on the UserRecordIdentifier
         AuthenticatedUser testAuthenticatedUser = new AuthenticatedUser();
@@ -160,12 +140,7 @@ public void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthE
 
     }
     @Test
-    @JvmSetting(key = JvmSettings.FEATURE_FLAG, value = "true", varArgs = "api-bearer-auth")
-    public void testFindUserFromRequest_oneProvider_validToken_noAccount() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
-
-        sut.userSvc = Mockito.mock(UserServiceBean.class);
-        sut.authSvc = Mockito.mock(AuthenticationServiceBean.class);
-
+    void testFindUserFromRequest_oneProvider_validToken_noAccount() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -176,7 +151,7 @@ public void testFindUserFromRequest_oneProvider_validToken_noAccount() throws Wr
         // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token
         UserRecordIdentifier userinfo = new UserRecordIdentifier(providerID, "KEY");
         BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY);
-        Mockito.when(oidcAuthProvider.getUserIdentifierForValidToken(token)).thenReturn(Optional.of(userinfo));
+        Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenReturn(Optional.of(userinfo));
 
         // ensures that the AuthenticationServiceBean can retrieve an Authenticated user based on the UserRecordIdentifier
         Mockito.when(sut.authSvc.lookupUser(userinfo)).thenReturn(null);

From 544a502e3f57b52a2dbaae562e43083e7a015315 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Sun, 21 May 2023 23:03:33 +0200
Subject: [PATCH 100/922] refactor(auth,api,test): adapt exception checks in
 BearerTokenAuthMechanismTest

- Now that we no longer need to catch OAuth2Exception, adapt accordingly
- This fixed the failing tests
---
 .../api/auth/BearerTokenAuthMechanismTest.java    | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
index 8a57ee4c41c..281f1d21d45 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/auth/BearerTokenAuthMechanismTest.java
@@ -6,7 +6,6 @@
 import edu.harvard.iq.dataverse.api.auth.doubles.BearerTokenKeyContainerRequestTestFake;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
-import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthProvider;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
@@ -50,7 +49,7 @@ void testFindUserFromRequest_no_token() throws WrappedAuthErrorResponse {
     }
 
     @Test
-    void testFindUserFromRequest_invalid_token() throws WrappedAuthErrorResponse {
+    void testFindUserFromRequest_invalid_token() {
         Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.emptySet());
         
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer ");
@@ -60,7 +59,7 @@ void testFindUserFromRequest_invalid_token() throws WrappedAuthErrorResponse {
         assertEquals(INVALID_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage());
     }
     @Test
-    void testFindUserFromRequest_no_OidcProvider() throws WrappedAuthErrorResponse {
+    void testFindUserFromRequest_no_OidcProvider() {
         Mockito.when(sut.authSvc.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Collections.emptySet());
         
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " +TEST_API_KEY);
@@ -71,7 +70,7 @@ void testFindUserFromRequest_no_OidcProvider() throws WrappedAuthErrorResponse {
     }
 
     @Test
-    void testFindUserFromRequest_oneProvider_invalidToken_1() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
+    void testFindUserFromRequest_oneProvider_invalidToken_1() throws ParseException, IOException {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -92,7 +91,7 @@ void testFindUserFromRequest_oneProvider_invalidToken_1() throws WrappedAuthErro
     }
 
     @Test
-    void testFindUserFromRequest_oneProvider_invalidToken_2() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
+    void testFindUserFromRequest_oneProvider_invalidToken_2() throws ParseException, IOException {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -102,7 +101,7 @@ void testFindUserFromRequest_oneProvider_invalidToken_2() throws WrappedAuthErro
 
         // ensure that the OIDCAuthProvider returns a valid UserRecordIdentifier for a given Token
         BearerAccessToken token = BearerAccessToken.parse("Bearer " + TEST_API_KEY);
-        Mockito.when(oidcAuthProvider.getUserIdentifierForValidToken(token)).thenThrow(OAuth2Exception.class);
+        Mockito.when(oidcAuthProvider.getUserIdentifier(token)).thenThrow(IOException.class);
 
         // when
         ContainerRequestContext testContainerRequest = new BearerTokenKeyContainerRequestTestFake("Bearer " + TEST_API_KEY);
@@ -112,7 +111,7 @@ void testFindUserFromRequest_oneProvider_invalidToken_2() throws WrappedAuthErro
         assertEquals(UNAUTHORIZED_BEARER_TOKEN, wrappedAuthErrorResponse.getMessage());
     }
     @Test
-    void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
+    void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthErrorResponse, ParseException, IOException {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);
@@ -140,7 +139,7 @@ void testFindUserFromRequest_oneProvider_validToken() throws WrappedAuthErrorRes
 
     }
     @Test
-    void testFindUserFromRequest_oneProvider_validToken_noAccount() throws WrappedAuthErrorResponse, ParseException, IOException, OAuth2Exception {
+    void testFindUserFromRequest_oneProvider_validToken_noAccount() throws WrappedAuthErrorResponse, ParseException, IOException {
         OIDCAuthProvider oidcAuthProvider = Mockito.mock(OIDCAuthProvider.class);
         String providerID = "OIEDC";
         Mockito.when(oidcAuthProvider.getId()).thenReturn(providerID);

From c859ef64adc6cb59065da4f0cf6aa0097a022701 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Sun, 21 May 2023 23:16:19 +0200
Subject: [PATCH 101/922] doc(test): add changes to JvmSetting helper in
 testing guide

---
 doc/sphinx-guides/source/developers/testing.rst | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index c734bed8b70..5814d9d4e7b 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -85,20 +85,26 @@ JUnit 5 Test Helper Extensions
 Our codebase provides little helpers to ease dealing with state during tests.
 Some tests might need to change something which should be restored after the test ran.
 
-For unit tests, the most interesting part is to set a JVM setting just for the current test.
-Please use the ``@JvmSetting(key = JvmSettings.XXX, value = "")`` annotation on a test method or
-a test class to set and clear the property automatically.
+For unit tests, the most interesting part is to set a JVM setting just for the current test or a whole test class.
+(Which might be an inner class, too!). Please make use of the ``@JvmSetting(key = JvmSettings.XXX, value = "")``
+annotation and also make sure to annotate the test class with ``@LocalJvmSettings``.
 
 Inspired by JUnit's ``@MethodSource`` annotation, you may use ``@JvmSetting(key = JvmSettings.XXX, method = "zzz")``
-to reference a method located in the same test class by name (i. e. ``private static String zzz() {}``) to allow
+to reference a static method located in the same test class by name (i. e. ``private static String zzz() {}``) to allow
 retrieving dynamic data instead of String constants only. (Note the requirement for a *static* method!)
 
+If you want to delete a setting, simply provide a ``null`` value. This can be used to override a class-wide setting
+or some other default that is present for some reason.
+
 To set arbitrary system properties for the current test, a similar extension ``@SystemProperty(key = "", value = "")``
 has been added. (Note: it does not support method references.)
 
 Both extensions will ensure the global state of system properties is non-interfering for
 test executions. Tests using these extensions will be executed in serial.
 
+This settings helper may be extended at a later time to manipulate settings in a remote instance during integration
+or end-to-end testing. Stay tuned!
+
 Observing Changes to Code Coverage
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

From bda75c18d5b1799e81f17f6711b9323441f559ff Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 01:46:52 +0200
Subject: [PATCH 102/922] feat(model): make UserRecordIdentifier comparable

Adding equals and hashCode methods to enable testing for equality in tests, etc.
---
 .../authorization/UserRecordIdentifier.java        | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/UserRecordIdentifier.java b/src/main/java/edu/harvard/iq/dataverse/authorization/UserRecordIdentifier.java
index 963ee592bbf..dfbb43fae46 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/UserRecordIdentifier.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/UserRecordIdentifier.java
@@ -2,6 +2,8 @@
 
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 
+import java.util.Objects;
+
 /**
  * Identifies a user using two strings:
  * <ul>
@@ -38,4 +40,16 @@ public AuthenticatedUserLookup createAuthenticatedUserLookup( AuthenticatedUser
         return new AuthenticatedUserLookup(userIdInRepo, repoId, u);
     }
     
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof UserRecordIdentifier)) return false;
+        UserRecordIdentifier that = (UserRecordIdentifier) o;
+        return Objects.equals(repoId, that.repoId) && Objects.equals(getUserIdInRepo(), that.getUserIdInRepo());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(repoId, getUserIdInRepo());
+    }
 }

From 7004191eeec6f2deb5a01d52222fa1da9cea725b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 01:52:38 +0200
Subject: [PATCH 103/922] feat(test,api,auth): create actual integration tests
 in OIDCAuthenticationProviderFactoryIT

- First test makes a roundtrip to receive the user info for the kcuser
- Second test simulates an API request with a bearer token
---
 .../OIDCAuthenticationProviderFactoryIT.java  | 123 +++++++++++++++++-
 1 file changed, 116 insertions(+), 7 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
index 53cfcca2742..a5aa29cc083 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -1,37 +1,146 @@
 package edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc;
 
+import com.nimbusds.oauth2.sdk.token.BearerAccessToken;
+import com.nimbusds.openid.connect.sdk.claims.UserInfo;
 import dasniko.testcontainers.keycloak.KeycloakContainer;
+import edu.harvard.iq.dataverse.UserServiceBean;
+import edu.harvard.iq.dataverse.api.auth.BearerTokenAuthMechanism;
+import edu.harvard.iq.dataverse.api.auth.doubles.BearerTokenKeyContainerRequestTestFake;
+import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
+import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.authorization.users.User;
+import edu.harvard.iq.dataverse.mocks.MockAuthenticatedUser;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
+import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.keycloak.admin.client.Keycloak;
+import org.keycloak.admin.client.KeycloakBuilder;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.Optional;
+import java.util.Set;
+
+import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientId;
+import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientSecret;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+import static org.mockito.Mockito.when;
 
 @Tag("testcontainers")
 @Testcontainers
+@ExtendWith(MockitoExtension.class)
+// NOTE: order is important here - Testcontainers must be first, otherwise it's not ready when we call getAuthUrl()
+@LocalJvmSettings
+@JvmSetting(key = JvmSettings.OIDC_CLIENT_ID, value = clientId)
+@JvmSetting(key = JvmSettings.OIDC_CLIENT_SECRET, value = clientSecret)
+@JvmSetting(key = JvmSettings.OIDC_AUTH_SERVER_URL, method = "getAuthUrl")
 class OIDCAuthenticationProviderFactoryIT {
     
+    // NOTE: the following values are taken from the realm import file!
     static final String clientId = "oidc-client";
     static final String clientSecret = "ss6gE8mODCDfqesQaSG3gwUwZqZt547E";
     static final String realm = "oidc-realm";
+    static final String adminUser = "kcuser";
+    static final String adminPassword = "kcpassword";
+    static final String clientIdAdminCli = "admin-cli";
     
+    // The realm JSON resides in conf/keycloak/oidc-realm.json and gets avail here using <testResources> in pom.xml
     @Container
-    static KeycloakContainer keycloakContainer = new KeycloakContainer().withRealmImportFile("keycloak/oidc-realm.json");
+    static KeycloakContainer keycloakContainer = new KeycloakContainer("quay.io/keycloak/keycloak:19.0")
+        .withRealmImportFile("keycloak/oidc-realm.json")
+        .withAdminUsername(adminUser)
+        .withAdminPassword(adminPassword);
     
-    // simple method to retrieve the issuer URL, referenced to by @JvmSetting annotations
+    // simple method to retrieve the issuer URL, referenced to by @JvmSetting annotations (do no delete)
     private static String getAuthUrl() {
         return keycloakContainer.getAuthServerUrl() + "realms/" + realm;
     }
     
+    OIDCAuthProvider getProvider() throws Exception {
+        OIDCAuthProvider oidcAuthProvider = (OIDCAuthProvider) OIDCAuthenticationProviderFactory.buildFromSettings();
+        
+        assumeTrue(oidcAuthProvider.getMetadata().getTokenEndpointURI().toString()
+            .startsWith(keycloakContainer.getAuthServerUrl()));
+        
+        return oidcAuthProvider;
+    }
+    
+    Keycloak getAdminClient() {
+        return KeycloakBuilder.builder()
+            .serverUrl(keycloakContainer.getAuthServerUrl())
+            .realm(realm)
+            .clientId(clientIdAdminCli)
+            .username(keycloakContainer.getAdminUsername())
+            .password(keycloakContainer.getAdminPassword())
+            .build();
+    }
+    
+    String getBearerToken() throws Exception {
+        Keycloak keycloak = getAdminClient();
+        return keycloak.tokenManager().getAccessTokenString();
+    }
+    
     @Test
-    @JvmSetting(key = JvmSettings.OIDC_CLIENT_ID, value = clientId)
-    @JvmSetting(key = JvmSettings.OIDC_CLIENT_SECRET, value = clientSecret)
-    @JvmSetting(key = JvmSettings.OIDC_AUTH_SERVER_URL, method = "getAuthUrl")
     void testCreateProvider() throws Exception {
-        OIDCAuthProvider oidcAuthProvider = (OIDCAuthProvider) OIDCAuthenticationProviderFactory.buildFromSettings();
-        assertTrue(oidcAuthProvider.getMetadata().getTokenEndpointURI().toString().startsWith(keycloakContainer.getAuthServerUrl()));
+        OIDCAuthProvider oidcAuthProvider = getProvider();
+        String token = getBearerToken();
+        assumeFalse(token == null);
+        
+        Optional<UserInfo> info = oidcAuthProvider.getUserInfo(new BearerAccessToken(token));
+        
+        assertTrue(info.isPresent());
+        assertEquals(adminUser, info.get().getPreferredUsername());
+    }
+    
+    @Mock
+    UserServiceBean userService;
+    @Mock
+    AuthenticationServiceBean authService;
+    
+    @InjectMocks
+    BearerTokenAuthMechanism bearerTokenAuthMechanism;
+    
+    @Test
+    @JvmSetting(key = JvmSettings.FEATURE_FLAG, varArgs = "api-bearer-auth", value = "true")
+    void testApiBearerAuth() throws Exception {
+        assumeFalse(userService == null);
+        assumeFalse(authService == null);
+        assumeFalse(bearerTokenAuthMechanism == null);
+        
+        // given
+        // Get the access token from the remote Keycloak in the container
+        String accessToken = getBearerToken();
+        assumeFalse(accessToken == null);
+        
+        OIDCAuthProvider oidcAuthProvider = getProvider();
+        // This will also receive the details from the remote Keycloak in the container
+        UserRecordIdentifier identifier = oidcAuthProvider.getUserIdentifier(new BearerAccessToken(accessToken)).get();
+        String token = "Bearer " + accessToken;
+        BearerTokenKeyContainerRequestTestFake request = new BearerTokenKeyContainerRequestTestFake(token);
+        AuthenticatedUser user = new MockAuthenticatedUser();
+        
+        // setup mocks (we don't want or need a database here)
+        when(authService.getAuthenticationProviderIdsOfType(OIDCAuthProvider.class)).thenReturn(Set.of(oidcAuthProvider.getId()));
+        when(authService.getAuthenticationProvider(oidcAuthProvider.getId())).thenReturn(oidcAuthProvider);
+        when(authService.lookupUser(identifier)).thenReturn(user);
+        when(userService.updateLastApiUseTime(user)).thenReturn(user);
+        
+        // when (let's do this again, but now with the actual subject under test!)
+        User lookedUpUser = bearerTokenAuthMechanism.findUserFromRequest(request);
+        
+        // then
+        assertNotNull(lookedUpUser);
+        assertEquals(user, lookedUpUser);
     }
 }
\ No newline at end of file

From 564d6a73bcc0e101299c7f370bdf5fc6d42f8287 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 01:53:34 +0200
Subject: [PATCH 104/922] feat(build): make integration test using
 Testcontainers executable via Maven

Simply call mvn verify to execute, as it is meant to be!
---
 pom.xml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pom.xml b/pom.xml
index 8764e4f493d..adda4bb31f5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -729,6 +729,22 @@
                     <skip>${skipUnitTests}</skip>
                 </configuration>
             </plugin>
+            <!-- Run mvn verify to execute these Testcontainers based integration tests. Needs Docker! -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <configuration>
+                    <groups>testcontainers</groups>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>integration-test</goal>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-checkstyle-plugin</artifactId>

From c207b3baa95a01c9d67cefd1cacaf5784bd914ff Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 01:53:57 +0200
Subject: [PATCH 105/922] chore(build): update Keycloak Testcontainers version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index adda4bb31f5..313f33b94b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -577,7 +577,7 @@
         <dependency>
             <groupId>com.github.dasniko</groupId>
             <artifactId>testcontainers-keycloak</artifactId>
-            <version>2.4.0</version>
+            <version>2.5.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>

From 7f8225f93c2353deb3f13e515ef1f43e9b0630c4 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 01:56:13 +0200
Subject: [PATCH 106/922] chore(build,test): add temporary servlet dependency
 in test scope

This is necessary because the Jakarta EE 8.0.0 API package did not properly
ship the bundle files necessary for servlet execution.

Not including this testing dependency leads to very cryptic errors ala
MissingResourceException for Bundle with lang en-US.

This should be removed once we migrate to Jakarta EE 10.
---
 pom.xml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/pom.xml b/pom.xml
index 313f33b94b8..e9a9b9dd611 100644
--- a/pom.xml
+++ b/pom.xml
@@ -580,6 +580,18 @@
             <version>2.5.0</version>
             <scope>test</scope>
         </dependency>
+        <!--
+            Brute force solution until we are on Jakarta EE 10.
+            Otherwise, we get very cryptic errors about missing bundle files on test runs.
+            See also https://github.com/jakartaee/jakartaee-api/issues/61
+        -->
+        <dependency>
+            <groupId>jakarta.servlet</groupId>
+            <artifactId>jakarta.servlet-api</artifactId>
+            <version>4.0.4</version>
+            <scope>test</scope>
+        </dependency>
+        
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>

From ac6354645c2bc8557e4cab19d2d28304605e634b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 16:09:29 +0200
Subject: [PATCH 107/922] fix(test,oidc): replace Keycloak Demo/Dev realm

The realm used before (created with Keycloak 16) was for some reason not compatible with Keycloak 20+.
A new "Test" realm was created with more users and working with Keycloak 20 and 21.

All files to run Keycloak have been adapted in version and realm import file.
---
 conf/keycloak/docker-compose.yml              |   12 +-
 .../keycloak/oidc-keycloak-auth-provider.json |    2 +-
 conf/keycloak/oidc-realm.json                 | 2108 -----------------
 conf/keycloak/run-keycloak.sh                 |    4 +-
 conf/keycloak/test-realm.json                 | 1939 +++++++++++++++
 docker-compose-dev.yml                        |    6 +-
 6 files changed, 1952 insertions(+), 2119 deletions(-)
 delete mode 100644 conf/keycloak/oidc-realm.json
 create mode 100644 conf/keycloak/test-realm.json

diff --git a/conf/keycloak/docker-compose.yml b/conf/keycloak/docker-compose.yml
index 2776f6572df..12b2382bd3d 100644
--- a/conf/keycloak/docker-compose.yml
+++ b/conf/keycloak/docker-compose.yml
@@ -3,13 +3,15 @@ version: "3.9"
 services:
 
   keycloak:
-    image: 'jboss/keycloak:16.1.1'
+    image: 'quay.io/keycloak/keycloak:21.0'
+    command:
+      - "start-dev"
+      - "--import-realm"
     environment:
-      - KEYCLOAK_USER=kcadmin
-      - KEYCLOAK_PASSWORD=kcpassword
-      - KEYCLOAK_IMPORT=/tmp/oidc-realm.json
+      - KEYCLOAK_ADMIN=kcadmin
+      - KEYCLOAK_ADMIN_PASSWORD=kcpassword
       - KEYCLOAK_LOGLEVEL=DEBUG
     ports:
       - "8090:8080"
     volumes:
-      - './oidc-realm.json:/tmp/oidc-realm.json'
+      - './test-realm.json:/opt/keycloak/data/import/test-realm.json'
diff --git a/conf/keycloak/oidc-keycloak-auth-provider.json b/conf/keycloak/oidc-keycloak-auth-provider.json
index 7d09fe5f36e..7e01bd4c325 100644
--- a/conf/keycloak/oidc-keycloak-auth-provider.json
+++ b/conf/keycloak/oidc-keycloak-auth-provider.json
@@ -3,6 +3,6 @@
   "factoryAlias": "oidc",
   "title": "OIDC-Keycloak",
   "subtitle": "OIDC-Keycloak",
-  "factoryData": "type: oidc | issuer: http://keycloak.mydomain.com:8090/realms/oidc-realm | clientId: oidc-client | clientSecret: ss6gE8mODCDfqesQaSG3gwUwZqZt547E",
+  "factoryData": "type: oidc | issuer: http://keycloak.mydomain.com:8090/realms/test | clientId: test | clientSecret: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8",
   "enabled": true
 }
diff --git a/conf/keycloak/oidc-realm.json b/conf/keycloak/oidc-realm.json
deleted file mode 100644
index 1b77f2b4384..00000000000
--- a/conf/keycloak/oidc-realm.json
+++ /dev/null
@@ -1,2108 +0,0 @@
-{
-  "id": "oidc-realm",
-  "realm": "oidc-realm",
-  "notBefore": 0,
-  "defaultSignatureAlgorithm": "RS256",
-  "revokeRefreshToken": false,
-  "refreshTokenMaxReuse": 0,
-  "accessTokenLifespan": 300,
-  "accessTokenLifespanForImplicitFlow": 900,
-  "ssoSessionIdleTimeout": 1800,
-  "ssoSessionMaxLifespan": 36000,
-  "ssoSessionIdleTimeoutRememberMe": 0,
-  "ssoSessionMaxLifespanRememberMe": 0,
-  "offlineSessionIdleTimeout": 2592000,
-  "offlineSessionMaxLifespanEnabled": false,
-  "offlineSessionMaxLifespan": 5184000,
-  "clientSessionIdleTimeout": 0,
-  "clientSessionMaxLifespan": 0,
-  "clientOfflineSessionIdleTimeout": 0,
-  "clientOfflineSessionMaxLifespan": 0,
-  "accessCodeLifespan": 60,
-  "accessCodeLifespanUserAction": 300,
-  "accessCodeLifespanLogin": 1800,
-  "actionTokenGeneratedByAdminLifespan": 43200,
-  "actionTokenGeneratedByUserLifespan": 300,
-  "oauth2DeviceCodeLifespan": 600,
-  "oauth2DevicePollingInterval": 5,
-  "enabled": true,
-  "sslRequired": "external",
-  "registrationAllowed": false,
-  "registrationEmailAsUsername": false,
-  "rememberMe": false,
-  "verifyEmail": false,
-  "loginWithEmailAllowed": true,
-  "duplicateEmailsAllowed": false,
-  "resetPasswordAllowed": false,
-  "editUsernameAllowed": false,
-  "bruteForceProtected": false,
-  "permanentLockout": false,
-  "maxFailureWaitSeconds": 900,
-  "minimumQuickLoginWaitSeconds": 60,
-  "waitIncrementSeconds": 60,
-  "quickLoginCheckMilliSeconds": 1000,
-  "maxDeltaTimeSeconds": 43200,
-  "failureFactor": 30,
-  "roles": {
-    "realm": [
-      {
-        "id": "13d76240-fcf8-4361-9dbf-de268717cfb2",
-        "name": "uma_authorization",
-        "description": "${role_uma_authorization}",
-        "composite": false,
-        "clientRole": false,
-        "containerId": "oidc-realm",
-        "attributes": {}
-      },
-      {
-        "id": "88b414c4-3516-4486-8f8b-a811ed0e0ce5",
-        "name": "default-roles-oidc-realm",
-        "description": "${role_default-roles}",
-        "composite": true,
-        "composites": {
-          "realm": [
-            "offline_access",
-            "uma_authorization"
-          ]
-        },
-        "clientRole": false,
-        "containerId": "oidc-realm",
-        "attributes": {}
-      },
-      {
-        "id": "b907fd4e-0e54-461c-9411-3f736eef7d2f",
-        "name": "offline_access",
-        "description": "${role_offline-access}",
-        "composite": false,
-        "clientRole": false,
-        "containerId": "oidc-realm",
-        "attributes": {}
-      }
-    ],
-    "client": {
-      "realm-management": [
-        {
-          "id": "39342ea9-0b4e-4841-8996-433759e9297f",
-          "name": "create-client",
-          "description": "${role_create-client}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "f8680034-617d-45d3-9801-7bf0d704c549",
-          "name": "manage-users",
-          "description": "${role_manage-users}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "b08e4cc3-71e2-4395-b66b-fb1277b48b88",
-          "name": "manage-realm",
-          "description": "${role_manage-realm}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "c15dc407-d012-43af-9a21-a2923e1d7b74",
-          "name": "manage-events",
-          "description": "${role_manage-events}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "66c07cb7-42cd-4155-8485-6cc7bd37cba9",
-          "name": "view-realm",
-          "description": "${role_view-realm}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "0419515f-4ab8-43ca-ac69-e842195813c0",
-          "name": "view-events",
-          "description": "${role_view-events}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "aa553d5a-b2dc-4f81-979a-2af0a019fee0",
-          "name": "impersonation",
-          "description": "${role_impersonation}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "9567e1e9-b755-43a8-93ed-d5929391316f",
-          "name": "manage-clients",
-          "description": "${role_manage-clients}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "e3dab69f-7323-4aad-bf98-8b7697f36d57",
-          "name": "query-users",
-          "description": "${role_query-users}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "ee8a4855-d0d5-4261-bdba-b419d304a824",
-          "name": "query-groups",
-          "description": "${role_query-groups}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "4f251212-e922-4ac0-9cce-3ada607648d2",
-          "name": "view-identity-providers",
-          "description": "${role_view-identity-providers}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "34e1dc59-a975-424f-887b-52465e184a4b",
-          "name": "realm-admin",
-          "description": "${role_realm-admin}",
-          "composite": true,
-          "composites": {
-            "client": {
-              "realm-management": [
-                "create-client",
-                "manage-users",
-                "manage-realm",
-                "manage-events",
-                "view-realm",
-                "view-events",
-                "impersonation",
-                "manage-clients",
-                "query-users",
-                "view-identity-providers",
-                "query-groups",
-                "view-clients",
-                "view-users",
-                "manage-authorization",
-                "manage-identity-providers",
-                "query-realms",
-                "query-clients",
-                "view-authorization"
-              ]
-            }
-          },
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "d35aca04-0182-40d3-96b8-1ce5cc118729",
-          "name": "view-clients",
-          "description": "${role_view-clients}",
-          "composite": true,
-          "composites": {
-            "client": {
-              "realm-management": [
-                "query-clients"
-              ]
-            }
-          },
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "7d3b28d5-471a-4b2b-bc80-56d4ff80fd28",
-          "name": "view-users",
-          "description": "${role_view-users}",
-          "composite": true,
-          "composites": {
-            "client": {
-              "realm-management": [
-                "query-users",
-                "query-groups"
-              ]
-            }
-          },
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "651059eb-fc1a-4f8d-9ced-ed28b0a2f965",
-          "name": "manage-authorization",
-          "description": "${role_manage-authorization}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "73f447e9-def8-4214-8516-56571f2c6f65",
-          "name": "manage-identity-providers",
-          "description": "${role_manage-identity-providers}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "1b5f7c39-885e-4246-8cf5-25769544fc3d",
-          "name": "query-realms",
-          "description": "${role_query-realms}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "350da4c1-69d4-4557-a9a8-8ba760db0225",
-          "name": "query-clients",
-          "description": "${role_query-clients}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        },
-        {
-          "id": "43d51082-6922-4765-8022-529d91a4603f",
-          "name": "view-authorization",
-          "description": "${role_view-authorization}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-          "attributes": {}
-        }
-      ],
-      "security-admin-console": [],
-      "admin-cli": [],
-      "account-console": [],
-      "broker": [],
-      "oidc-client": [],
-      "account": [
-        {
-          "id": "a163535c-71de-4b2d-9530-26b25eeb1c1e",
-          "name": "delete-account",
-          "description": "${role_delete-account}",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "aed2e103-ee29-4d5c-a34e-1b8c65b7d537",
-          "attributes": {}
-        },
-        {
-          "id": "851c6a9f-bce7-4c70-be82-084c25d61b25",
-          "name": "manage-account",
-          "composite": false,
-          "clientRole": true,
-          "containerId": "aed2e103-ee29-4d5c-a34e-1b8c65b7d537",
-          "attributes": {}
-        }
-      ]
-    }
-  },
-  "groups": [],
-  "defaultRole": {
-    "id": "88b414c4-3516-4486-8f8b-a811ed0e0ce5",
-    "name": "default-roles-oidc-realm",
-    "description": "${role_default-roles}",
-    "composite": true,
-    "clientRole": false,
-    "containerId": "oidc-realm"
-  },
-  "requiredCredentials": [
-    "password"
-  ],
-  "otpPolicyType": "totp",
-  "otpPolicyAlgorithm": "HmacSHA1",
-  "otpPolicyInitialCounter": 0,
-  "otpPolicyDigits": 6,
-  "otpPolicyLookAheadWindow": 1,
-  "otpPolicyPeriod": 30,
-  "otpSupportedApplications": [
-    "FreeOTP",
-    "Google Authenticator"
-  ],
-  "webAuthnPolicyRpEntityName": "keycloak",
-  "webAuthnPolicySignatureAlgorithms": [
-    "ES256"
-  ],
-  "webAuthnPolicyRpId": "",
-  "webAuthnPolicyAttestationConveyancePreference": "not specified",
-  "webAuthnPolicyAuthenticatorAttachment": "not specified",
-  "webAuthnPolicyRequireResidentKey": "not specified",
-  "webAuthnPolicyUserVerificationRequirement": "not specified",
-  "webAuthnPolicyCreateTimeout": 0,
-  "webAuthnPolicyAvoidSameAuthenticatorRegister": false,
-  "webAuthnPolicyAcceptableAaguids": [],
-  "webAuthnPolicyPasswordlessRpEntityName": "keycloak",
-  "webAuthnPolicyPasswordlessSignatureAlgorithms": [
-    "ES256"
-  ],
-  "webAuthnPolicyPasswordlessRpId": "",
-  "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified",
-  "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified",
-  "webAuthnPolicyPasswordlessRequireResidentKey": "not specified",
-  "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified",
-  "webAuthnPolicyPasswordlessCreateTimeout": 0,
-  "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false,
-  "webAuthnPolicyPasswordlessAcceptableAaguids": [],
-  "users": [
-    {
-      "username": "kcuser",
-      "enabled": true,
-      "totp": false,
-      "emailVerified": true,
-      "firstName": "Test",
-      "lastName": "Test",
-      "email": "test@test.com",
-      "credentials": [
-        {
-          "type": "password",
-          "value": "kcpassword"
-        }
-      ]
-    }
-  ],
-  "scopeMappings": [
-    {
-      "clientScope": "offline_access",
-      "roles": [
-        "offline_access"
-      ]
-    }
-  ],
-  "clientScopeMappings": {
-    "account": [
-      {
-        "client": "account-console",
-        "roles": [
-          "manage-account"
-        ]
-      }
-    ]
-  },
-  "clients": [
-    {
-      "id": "aed2e103-ee29-4d5c-a34e-1b8c65b7d537",
-      "clientId": "account",
-      "name": "${client_account}",
-      "rootUrl": "${authBaseUrl}",
-      "baseUrl": "/realms/oidc-realm/account/",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "redirectUris": [
-        "/realms/oidc-realm/account/*"
-      ],
-      "webOrigins": [],
-      "notBefore": 0,
-      "bearerOnly": false,
-      "consentRequired": false,
-      "standardFlowEnabled": true,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": false,
-      "serviceAccountsEnabled": false,
-      "publicClient": true,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {},
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": false,
-      "nodeReRegistrationTimeout": 0,
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    },
-    {
-      "id": "1e821c0e-f6b9-4324-9b23-e82b5431fb72",
-      "clientId": "account-console",
-      "name": "${client_account-console}",
-      "rootUrl": "${authBaseUrl}",
-      "baseUrl": "/realms/oidc-realm/account/",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "redirectUris": [
-        "/realms/oidc-realm/account/*"
-      ],
-      "webOrigins": [],
-      "notBefore": 0,
-      "bearerOnly": false,
-      "consentRequired": false,
-      "standardFlowEnabled": true,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": false,
-      "serviceAccountsEnabled": false,
-      "publicClient": true,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {
-        "pkce.code.challenge.method": "S256"
-      },
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": false,
-      "nodeReRegistrationTimeout": 0,
-      "protocolMappers": [
-        {
-          "id": "397616ab-4124-4a13-92b6-317423e818a3",
-          "name": "audience resolve",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-audience-resolve-mapper",
-          "consentRequired": false,
-          "config": {}
-        }
-      ],
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    },
-    {
-      "id": "dddcc3e0-d742-422b-8b5f-84a292ea9d66",
-      "clientId": "admin-cli",
-      "name": "${client_admin-cli}",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "redirectUris": [],
-      "webOrigins": [],
-      "notBefore": 0,
-      "bearerOnly": false,
-      "consentRequired": false,
-      "standardFlowEnabled": false,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": true,
-      "serviceAccountsEnabled": false,
-      "publicClient": true,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {},
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": false,
-      "nodeReRegistrationTimeout": 0,
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    },
-    {
-      "id": "df6f6cd0-a046-492f-84ac-b4fe31909be4",
-      "clientId": "broker",
-      "name": "${client_broker}",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "redirectUris": [],
-      "webOrigins": [],
-      "notBefore": 0,
-      "bearerOnly": true,
-      "consentRequired": false,
-      "standardFlowEnabled": true,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": false,
-      "serviceAccountsEnabled": false,
-      "publicClient": false,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {},
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": false,
-      "nodeReRegistrationTimeout": 0,
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    },
-    {
-      "id": "c0af31b9-21aa-4e70-baf3-8d68850c4081",
-      "clientId": "oidc-client",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "secret": "ss6gE8mODCDfqesQaSG3gwUwZqZt547E",
-      "redirectUris": [
-        "*"
-      ],
-      "webOrigins": [
-        "+"
-      ],
-      "notBefore": 0,
-      "bearerOnly": false,
-      "consentRequired": false,
-      "standardFlowEnabled": true,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": false,
-      "serviceAccountsEnabled": false,
-      "publicClient": false,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {
-        "saml.force.post.binding": "false",
-        "saml.multivalued.roles": "false",
-        "oauth2.device.authorization.grant.enabled": "false",
-        "use.jwks.url": "true",
-        "backchannel.logout.revoke.offline.tokens": "false",
-        "saml.server.signature.keyinfo.ext": "false",
-        "use.refresh.tokens": "true",
-        "jwt.credential.certificate": "MIICpTCCAY0CBgGE8V6o6TANBgkqhkiG9w0BAQsFADAWMRQwEgYDVQQDDAtvaWRjLWNsaWVudDAeFw0yMjEyMDgxMDUyMDNaFw0zMjEyMDgxMDUzNDNaMBYxFDASBgNVBAMMC29pZGMtY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArUffTl+jXWzyY3T4VVtkiGyNnY+RgyAXUzz+dxT7wUQaYSiNPvmaxnio555pWjR403SRUjVxM8eJYgHK9s43qQWdheXBIHyLKaQfjVsTtSmHgFtPmjk+kweQs6fxUi5CNvtx4RTCaOK5wV8q5q1X7mb8cZ5+gLSx1f/pHtayFXMT75nV04aZKWgPztPz8w+QXUx9cuFY4OIiTdRbdyfr1oOiDtMbxxA22tggB/HSMVkSckT3LSPj7fJKJMPFYi/g1AXxGipX/q8XkmOBrvNePCpH0F/IZbC1vXEsDC6urfoijOdiZgPMobuADmWHPiw2zgCN8qa6QuLFaI+JduXT9QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCEOYRHkH8DnBucb+uN5c9U/fZY+mpglxzZvby7dGBXfVwLN+eP1kGcQPaFi+nshk7FgF4mR5/cmuAPZt+YBbgP0z37D49nB7S6sniwzfhCAAplOT4vmm+MjperTDsWFUGhQZJvN/jxqP2Xccw7N//ReYi7yOlmWhwGyqQyTi0ySbE3BY5eFvUKepekybYi/15XlyF8lwS2jH1MvnJAxAMNVpVUcP4wTnq/dOw5ybrVWF0mPnA8KVzTPuPE5nzZvZ3rkXQeEJTffIToR+T/DH/KTLXcNUtx4nG0ajJ0gM6iVAXGnKlI9Viq/M5Ese+52I6rQmxTsFMn57LNzKgMpWcE",
-        "oidc.ciba.grant.enabled": "false",
-        "use.jwks.string": "false",
-        "backchannel.logout.session.required": "false",
-        "client_credentials.use_refresh_token": "false",
-        "require.pushed.authorization.requests": "false",
-        "saml.client.signature": "false",
-        "id.token.as.detached.signature": "false",
-        "saml.assertion.signature": "false",
-        "saml.encrypt": "false",
-        "saml.server.signature": "false",
-        "exclude.session.state.from.auth.response": "false",
-        "saml.artifact.binding": "false",
-        "saml_force_name_id_format": "false",
-        "tls.client.certificate.bound.access.tokens": "false",
-        "saml.authnstatement": "false",
-        "display.on.consent.screen": "false",
-        "saml.onetimeuse.condition": "false"
-      },
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": true,
-      "nodeReRegistrationTimeout": -1,
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    },
-    {
-      "id": "43ffb712-f233-48e2-ae79-d6993bac34a5",
-      "clientId": "realm-management",
-      "name": "${client_realm-management}",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "redirectUris": [],
-      "webOrigins": [],
-      "notBefore": 0,
-      "bearerOnly": true,
-      "consentRequired": false,
-      "standardFlowEnabled": true,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": false,
-      "serviceAccountsEnabled": false,
-      "publicClient": false,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {},
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": false,
-      "nodeReRegistrationTimeout": 0,
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    },
-    {
-      "id": "3747f98f-efbb-49ef-8238-a349bf5ab409",
-      "clientId": "security-admin-console",
-      "name": "${client_security-admin-console}",
-      "rootUrl": "${authAdminUrl}",
-      "baseUrl": "/admin/oidc-realm/console/",
-      "surrogateAuthRequired": false,
-      "enabled": true,
-      "alwaysDisplayInConsole": false,
-      "clientAuthenticatorType": "client-secret",
-      "redirectUris": [
-        "/admin/oidc-realm/console/*"
-      ],
-      "webOrigins": [
-        "+"
-      ],
-      "notBefore": 0,
-      "bearerOnly": false,
-      "consentRequired": false,
-      "standardFlowEnabled": true,
-      "implicitFlowEnabled": false,
-      "directAccessGrantsEnabled": false,
-      "serviceAccountsEnabled": false,
-      "publicClient": true,
-      "frontchannelLogout": false,
-      "protocol": "openid-connect",
-      "attributes": {
-        "pkce.code.challenge.method": "S256"
-      },
-      "authenticationFlowBindingOverrides": {},
-      "fullScopeAllowed": false,
-      "nodeReRegistrationTimeout": 0,
-      "protocolMappers": [
-        {
-          "id": "2fbdf6c9-ee69-4edc-b780-ec62aecfc519",
-          "name": "locale",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "locale",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "locale",
-            "jsonType.label": "String"
-          }
-        }
-      ],
-      "defaultClientScopes": [
-        "web-origins",
-        "roles",
-        "profile",
-        "email"
-      ],
-      "optionalClientScopes": [
-        "address",
-        "phone",
-        "offline_access",
-        "microprofile-jwt"
-      ]
-    }
-  ],
-  "clientScopes": [
-    {
-      "id": "f76f507d-7d1c-495b-9504-47830b3834f1",
-      "name": "phone",
-      "description": "OpenID Connect built-in scope: phone",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "true",
-        "display.on.consent.screen": "true",
-        "consent.screen.text": "${phoneScopeConsentText}"
-      },
-      "protocolMappers": [
-        {
-          "id": "be849ec8-1747-4efb-bc00-beeaf44f11c8",
-          "name": "phone number verified",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "phoneNumberVerified",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "phone_number_verified",
-            "jsonType.label": "boolean"
-          }
-        },
-        {
-          "id": "8e8600ec-4290-435d-b109-9f0547cb4a1d",
-          "name": "phone number",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "phoneNumber",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "phone_number",
-            "jsonType.label": "String"
-          }
-        }
-      ]
-    },
-    {
-      "id": "54b87197-5309-4b2c-8ad9-f561a0fc178a",
-      "name": "role_list",
-      "description": "SAML role list",
-      "protocol": "saml",
-      "attributes": {
-        "consent.screen.text": "${samlRoleListScopeConsentText}",
-        "display.on.consent.screen": "true"
-      },
-      "protocolMappers": [
-        {
-          "id": "5fd831af-19a5-4a9c-b44f-2a806fae011c",
-          "name": "role list",
-          "protocol": "saml",
-          "protocolMapper": "saml-role-list-mapper",
-          "consentRequired": false,
-          "config": {
-            "single": "false",
-            "attribute.nameformat": "Basic",
-            "attribute.name": "Role"
-          }
-        }
-      ]
-    },
-    {
-      "id": "2f85470d-8cb7-4f07-8602-47342d68af86",
-      "name": "web-origins",
-      "description": "OpenID Connect scope for add allowed web origins to the access token",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "false",
-        "display.on.consent.screen": "false",
-        "consent.screen.text": ""
-      },
-      "protocolMappers": [
-        {
-          "id": "c5d2aafc-f72d-4d7b-9d88-cd759f0e045e",
-          "name": "allowed web origins",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-allowed-origins-mapper",
-          "consentRequired": false,
-          "config": {}
-        }
-      ]
-    },
-    {
-      "id": "528face9-229a-4adf-98d8-68b1a22e880d",
-      "name": "microprofile-jwt",
-      "description": "Microprofile - JWT built-in scope",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "true",
-        "display.on.consent.screen": "false"
-      },
-      "protocolMappers": [
-        {
-          "id": "89240a7c-10f3-4e09-9d6b-41955b86c58d",
-          "name": "groups",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-realm-role-mapper",
-          "consentRequired": false,
-          "config": {
-            "multivalued": "true",
-            "userinfo.token.claim": "true",
-            "user.attribute": "foo",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "groups",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "15b6db72-4870-480e-a675-87f87df5f8a5",
-          "name": "upn",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "username",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "upn",
-            "jsonType.label": "String"
-          }
-        }
-      ]
-    },
-    {
-      "id": "cdd11477-b02b-4886-bc6d-cf4b728ebc0e",
-      "name": "email",
-      "description": "OpenID Connect built-in scope: email",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "true",
-        "display.on.consent.screen": "true",
-        "consent.screen.text": "${emailScopeConsentText}"
-      },
-      "protocolMappers": [
-        {
-          "id": "627b9f4f-23d6-4480-adf4-264faf58de33",
-          "name": "email verified",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "emailVerified",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "email_verified",
-            "jsonType.label": "boolean"
-          }
-        },
-        {
-          "id": "6a2adf2e-db2d-4ebe-8d48-f658f9b4a5ca",
-          "name": "email",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "email",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "email",
-            "jsonType.label": "String"
-          }
-        }
-      ]
-    },
-    {
-      "id": "8f830142-b3f1-40f0-82e2-ceed68857a40",
-      "name": "roles",
-      "description": "OpenID Connect scope for add user roles to the access token",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "false",
-        "display.on.consent.screen": "true",
-        "consent.screen.text": "${rolesScopeConsentText}"
-      },
-      "protocolMappers": [
-        {
-          "id": "28a96dc6-c4dc-4aae-b316-28b56dccd077",
-          "name": "audience resolve",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-audience-resolve-mapper",
-          "consentRequired": false,
-          "config": {}
-        },
-        {
-          "id": "3e81050f-540e-4f3d-9abf-86406e484f76",
-          "name": "realm roles",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-realm-role-mapper",
-          "consentRequired": false,
-          "config": {
-            "user.attribute": "foo",
-            "access.token.claim": "true",
-            "claim.name": "realm_access.roles",
-            "jsonType.label": "String",
-            "multivalued": "true"
-          }
-        },
-        {
-          "id": "13afa1f4-3fac-4c90-a9b4-e84e682f46e9",
-          "name": "client roles",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-client-role-mapper",
-          "consentRequired": false,
-          "config": {
-            "user.attribute": "foo",
-            "access.token.claim": "true",
-            "claim.name": "resource_access.${client_id}.roles",
-            "jsonType.label": "String",
-            "multivalued": "true"
-          }
-        }
-      ]
-    },
-    {
-      "id": "3beac2fc-e947-408f-8422-ca9a1e66a258",
-      "name": "address",
-      "description": "OpenID Connect built-in scope: address",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "true",
-        "display.on.consent.screen": "true",
-        "consent.screen.text": "${addressScopeConsentText}"
-      },
-      "protocolMappers": [
-        {
-          "id": "12911891-db5c-4a35-80fa-555c5eda7e68",
-          "name": "address",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-address-mapper",
-          "consentRequired": false,
-          "config": {
-            "user.attribute.formatted": "formatted",
-            "user.attribute.country": "country",
-            "user.attribute.postal_code": "postal_code",
-            "userinfo.token.claim": "true",
-            "user.attribute.street": "street",
-            "id.token.claim": "true",
-            "user.attribute.region": "region",
-            "access.token.claim": "true",
-            "user.attribute.locality": "locality"
-          }
-        }
-      ]
-    },
-    {
-      "id": "8a29297a-e6f6-41ae-b25d-8a14236de535",
-      "name": "offline_access",
-      "description": "OpenID Connect built-in scope: offline_access",
-      "protocol": "openid-connect",
-      "attributes": {
-        "consent.screen.text": "${offlineAccessScopeConsentText}",
-        "display.on.consent.screen": "true"
-      }
-    },
-    {
-      "id": "ce1622c5-701f-4e3e-9d2d-8dae0f07a295",
-      "name": "profile",
-      "description": "OpenID Connect built-in scope: profile",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "true",
-        "display.on.consent.screen": "true",
-        "consent.screen.text": "${profileScopeConsentText}"
-      },
-      "protocolMappers": [
-        {
-          "id": "98cc62b8-250a-4087-92da-bb0f0931e675",
-          "name": "full name",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-full-name-mapper",
-          "consentRequired": false,
-          "config": {
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "userinfo.token.claim": "true"
-          }
-        },
-        {
-          "id": "b99c8c44-4cc9-4c87-a5a1-c14e64d472ae",
-          "name": "given name",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "firstName",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "given_name",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "903d5932-bdec-42bc-a53c-3cce93deaa1c",
-          "name": "zoneinfo",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "zoneinfo",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "zoneinfo",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "ccbdc095-28f7-4769-8261-2e32c7b6fab0",
-          "name": "picture",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "picture",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "picture",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "22a4a38c-f755-44f3-b847-803c7fb3cef5",
-          "name": "birthdate",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "birthdate",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "birthdate",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "78726920-b4e2-4ed2-b9e0-df38a7f82376",
-          "name": "updated at",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "updatedAt",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "updated_at",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "c64c6eb8-5cbe-4092-bf2c-dd02b8c0e0e8",
-          "name": "family name",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "lastName",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "family_name",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "306784d8-8da1-48d8-92a3-dccfff83bcaf",
-          "name": "middle name",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "middleName",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "middle_name",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "0ff127fa-774e-43a8-a1fc-47ea3f307aa1",
-          "name": "website",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "website",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "website",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "8989c6f8-25c5-4d02-aa06-25b3b77fc227",
-          "name": "profile",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "profile",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "profile",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "3b67000c-9cbf-43ee-9e05-26f560871897",
-          "name": "gender",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "gender",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "gender",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "c28b04de-2770-423e-9b9a-b3321d7300e2",
-          "name": "nickname",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "nickname",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "nickname",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "fd791ed4-d4ab-4df9-81b4-c69a3134bcab",
-          "name": "username",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "username",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "preferred_username",
-            "jsonType.label": "String"
-          }
-        },
-        {
-          "id": "c7378ce5-3673-47b2-9ebc-92c772bebf9f",
-          "name": "locale",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-attribute-mapper",
-          "consentRequired": false,
-          "config": {
-            "userinfo.token.claim": "true",
-            "user.attribute": "locale",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "claim.name": "locale",
-            "jsonType.label": "String"
-          }
-        }
-      ]
-    }
-  ],
-  "defaultDefaultClientScopes": [
-    "web-origins",
-    "role_list",
-    "roles",
-    "email",
-    "profile"
-  ],
-  "defaultOptionalClientScopes": [
-    "address",
-    "microprofile-jwt",
-    "offline_access",
-    "phone"
-  ],
-  "browserSecurityHeaders": {
-    "contentSecurityPolicyReportOnly": "",
-    "xContentTypeOptions": "nosniff",
-    "xRobotsTag": "none",
-    "xFrameOptions": "SAMEORIGIN",
-    "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';",
-    "xXSSProtection": "1; mode=block",
-    "strictTransportSecurity": "max-age=31536000; includeSubDomains"
-  },
-  "smtpServer": {},
-  "eventsEnabled": false,
-  "eventsListeners": [
-    "jboss-logging"
-  ],
-  "enabledEventTypes": [],
-  "adminEventsEnabled": false,
-  "adminEventsDetailsEnabled": false,
-  "identityProviders": [],
-  "identityProviderMappers": [],
-  "components": {
-    "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [
-      {
-        "id": "8e2d0c22-0627-4115-9f14-4225244333d9",
-        "name": "Trusted Hosts",
-        "providerId": "trusted-hosts",
-        "subType": "anonymous",
-        "subComponents": {},
-        "config": {
-          "host-sending-registration-request-must-match": [
-            "true"
-          ],
-          "client-uris-must-match": [
-            "true"
-          ]
-        }
-      },
-      {
-        "id": "45bdde87-a364-4d66-a12e-1a4fd42c85fb",
-        "name": "Full Scope Disabled",
-        "providerId": "scope",
-        "subType": "anonymous",
-        "subComponents": {},
-        "config": {}
-      },
-      {
-        "id": "7b7d3215-68d2-41db-bc0f-db0a45934a84",
-        "name": "Allowed Client Scopes",
-        "providerId": "allowed-client-templates",
-        "subType": "anonymous",
-        "subComponents": {},
-        "config": {
-          "allow-default-scopes": [
-            "true"
-          ]
-        }
-      },
-      {
-        "id": "e067781a-6058-4f2b-9408-3390e9854cf8",
-        "name": "Consent Required",
-        "providerId": "consent-required",
-        "subType": "anonymous",
-        "subComponents": {},
-        "config": {}
-      },
-      {
-        "id": "296be954-8084-45c8-b6f3-94d53f7341f6",
-        "name": "Allowed Protocol Mapper Types",
-        "providerId": "allowed-protocol-mappers",
-        "subType": "anonymous",
-        "subComponents": {},
-        "config": {
-          "allowed-protocol-mapper-types": [
-            "saml-role-list-mapper",
-            "saml-user-property-mapper",
-            "oidc-usermodel-attribute-mapper",
-            "oidc-address-mapper",
-            "oidc-sha256-pairwise-sub-mapper",
-            "saml-user-attribute-mapper",
-            "oidc-usermodel-property-mapper",
-            "oidc-full-name-mapper"
-          ]
-        }
-      },
-      {
-        "id": "b9a2a484-aee1-4633-aa37-a9ab2b74a239",
-        "name": "Allowed Client Scopes",
-        "providerId": "allowed-client-templates",
-        "subType": "authenticated",
-        "subComponents": {},
-        "config": {
-          "allow-default-scopes": [
-            "true"
-          ]
-        }
-      },
-      {
-        "id": "016e4914-a32c-40fa-8aab-3eb25a411df5",
-        "name": "Max Clients Limit",
-        "providerId": "max-clients",
-        "subType": "anonymous",
-        "subComponents": {},
-        "config": {
-          "max-clients": [
-            "200"
-          ]
-        }
-      },
-      {
-        "id": "a4fb2fa3-93b8-4497-8047-424f70f298c7",
-        "name": "Allowed Protocol Mapper Types",
-        "providerId": "allowed-protocol-mappers",
-        "subType": "authenticated",
-        "subComponents": {},
-        "config": {
-          "allowed-protocol-mapper-types": [
-            "oidc-sha256-pairwise-sub-mapper",
-            "oidc-full-name-mapper",
-            "saml-user-property-mapper",
-            "saml-role-list-mapper",
-            "oidc-usermodel-attribute-mapper",
-            "oidc-address-mapper",
-            "oidc-usermodel-property-mapper",
-            "saml-user-attribute-mapper"
-          ]
-        }
-      }
-    ],
-    "org.keycloak.keys.KeyProvider": [
-      {
-        "id": "31b693fa-2b95-47a6-96a1-dfff868ca1df",
-        "name": "rsa-enc-generated",
-        "providerId": "rsa-enc-generated",
-        "subComponents": {},
-        "config": {
-          "priority": [
-            "100"
-          ],
-          "algorithm": [
-            "RSA-OAEP"
-          ]
-        }
-      },
-      {
-        "id": "f1e63d09-45a0-4382-8346-0408ee906649",
-        "name": "hmac-generated",
-        "providerId": "hmac-generated",
-        "subComponents": {},
-        "config": {
-          "priority": [
-            "100"
-          ],
-          "algorithm": [
-            "HS256"
-          ]
-        }
-      },
-      {
-        "id": "99084d92-06f5-4787-b932-a40b5377f3cb",
-        "name": "rsa-generated",
-        "providerId": "rsa-generated",
-        "subComponents": {},
-        "config": {
-          "priority": [
-            "100"
-          ]
-        }
-      },
-      {
-        "id": "9887f1bf-b4f7-4646-9919-a9dbde13ce74",
-        "name": "aes-generated",
-        "providerId": "aes-generated",
-        "subComponents": {},
-        "config": {
-          "priority": [
-            "100"
-          ]
-        }
-      }
-    ]
-  },
-  "internationalizationEnabled": false,
-  "supportedLocales": [],
-  "authenticationFlows": [
-    {
-      "id": "a7f91199-178d-4399-8319-5063ffcc37b0",
-      "alias": "Account verification options",
-      "description": "Method with which to verity the existing account",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "idp-email-verification",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "ALTERNATIVE",
-          "priority": 20,
-          "flowAlias": "Verify Existing Account by Re-authentication",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "602533e3-f7a1-4e25-9a12-f3080eeccec3",
-      "alias": "Authentication Options",
-      "description": "Authentication options.",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "basic-auth",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "basic-auth-otp",
-          "authenticatorFlow": false,
-          "requirement": "DISABLED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "auth-spnego",
-          "authenticatorFlow": false,
-          "requirement": "DISABLED",
-          "priority": 30,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "ba7bcdfd-05c6-4da6-827b-24e3513bddbe",
-      "alias": "Browser - Conditional OTP",
-      "description": "Flow to determine if the OTP is required for the authentication",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "conditional-user-configured",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "auth-otp-form",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "d0f62327-ef2f-4561-8b5a-1f61faecdac0",
-      "alias": "Direct Grant - Conditional OTP",
-      "description": "Flow to determine if the OTP is required for the authentication",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "conditional-user-configured",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "direct-grant-validate-otp",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "f10b85d0-26ee-4648-b81b-80213b066d76",
-      "alias": "First broker login - Conditional OTP",
-      "description": "Flow to determine if the OTP is required for the authentication",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "conditional-user-configured",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "auth-otp-form",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "d6af4ac0-f6bc-4197-bf01-6e2c321ecaad",
-      "alias": "Handle Existing Account",
-      "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "idp-confirm-link",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "flowAlias": "Account verification options",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "501ab743-2e2f-427d-820f-14deed111b08",
-      "alias": "Reset - Conditional OTP",
-      "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "conditional-user-configured",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "reset-otp",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "e02c3a63-a09d-4dde-9f6c-22c95eef8534",
-      "alias": "User creation or linking",
-      "description": "Flow for the existing/non-existing user alternatives",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticatorConfig": "create unique user config",
-          "authenticator": "idp-create-user-if-unique",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "ALTERNATIVE",
-          "priority": 20,
-          "flowAlias": "Handle Existing Account",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "c348906d-6266-4e68-937e-8f3d15c66524",
-      "alias": "Verify Existing Account by Re-authentication",
-      "description": "Reauthentication of existing account",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "idp-username-password-form",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "CONDITIONAL",
-          "priority": 20,
-          "flowAlias": "First broker login - Conditional OTP",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "cf6ba166-43d5-4687-95c4-0a184ca08885",
-      "alias": "browser",
-      "description": "browser based authentication",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "auth-cookie",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "auth-spnego",
-          "authenticatorFlow": false,
-          "requirement": "DISABLED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "identity-provider-redirector",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 25,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "ALTERNATIVE",
-          "priority": 30,
-          "flowAlias": "forms",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "87cb4f25-9275-4617-9e95-63adf1ce3ece",
-      "alias": "clients",
-      "description": "Base authentication for clients",
-      "providerId": "client-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "client-secret",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "client-jwt",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "client-secret-jwt",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 30,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "client-x509",
-          "authenticatorFlow": false,
-          "requirement": "ALTERNATIVE",
-          "priority": 40,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "e75b99c5-c566-4009-b0ba-c73716bed254",
-      "alias": "direct grant",
-      "description": "OpenID Connect Resource Owner Grant",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "direct-grant-validate-username",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "direct-grant-validate-password",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "CONDITIONAL",
-          "priority": 30,
-          "flowAlias": "Direct Grant - Conditional OTP",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "8a97380c-0f70-45cb-a7b0-780eb70453ba",
-      "alias": "docker auth",
-      "description": "Used by Docker clients to authenticate against the IDP",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "docker-http-basic-authenticator",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "131e0aad-5422-4504-bafc-96be2fa44c34",
-      "alias": "first broker login",
-      "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticatorConfig": "review profile config",
-          "authenticator": "idp-review-profile",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "flowAlias": "User creation or linking",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "e7d4b793-b3c2-4ec3-a2b1-04f7217e8f46",
-      "alias": "forms",
-      "description": "Username, password, otp and other auth forms.",
-      "providerId": "basic-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "auth-username-password-form",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "CONDITIONAL",
-          "priority": 20,
-          "flowAlias": "Browser - Conditional OTP",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "f59a7688-61a1-4ac9-a13a-03f92e022add",
-      "alias": "http challenge",
-      "description": "An authentication flow based on challenge-response HTTP Authentication Schemes",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "no-cookie-redirect",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "flowAlias": "Authentication Options",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "80a7b0f5-abb3-4780-be58-4ed1dc3e50fa",
-      "alias": "registration",
-      "description": "registration flow",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "registration-page-form",
-          "authenticatorFlow": true,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "flowAlias": "registration form",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "f18231cf-b803-493b-9dd6-ee8fa602c861",
-      "alias": "registration form",
-      "description": "registration form",
-      "providerId": "form-flow",
-      "topLevel": false,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "registration-user-creation",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "registration-profile-action",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 40,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "registration-password-action",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 50,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "registration-recaptcha-action",
-          "authenticatorFlow": false,
-          "requirement": "DISABLED",
-          "priority": 60,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    },
-    {
-      "id": "34ccfce6-1488-4db3-b90e-d98e8d8b2ae6",
-      "alias": "reset credentials",
-      "description": "Reset credentials for a user if they forgot their password or something",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "reset-credentials-choose-user",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "reset-credential-email",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 20,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticator": "reset-password",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 30,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        },
-        {
-          "authenticatorFlow": true,
-          "requirement": "CONDITIONAL",
-          "priority": 40,
-          "flowAlias": "Reset - Conditional OTP",
-          "userSetupAllowed": false,
-          "autheticatorFlow": true
-        }
-      ]
-    },
-    {
-      "id": "4468100c-fa83-4c16-8970-d53cb592f93a",
-      "alias": "saml ecp",
-      "description": "SAML ECP Profile Authentication Flow",
-      "providerId": "basic-flow",
-      "topLevel": true,
-      "builtIn": true,
-      "authenticationExecutions": [
-        {
-          "authenticator": "http-basic-authenticator",
-          "authenticatorFlow": false,
-          "requirement": "REQUIRED",
-          "priority": 10,
-          "userSetupAllowed": false,
-          "autheticatorFlow": false
-        }
-      ]
-    }
-  ],
-  "authenticatorConfig": [
-    {
-      "id": "c3bb087e-7fe9-4f13-b1bd-c2d7d1320054",
-      "alias": "create unique user config",
-      "config": {
-        "require.password.update.after.registration": "false"
-      }
-    },
-    {
-      "id": "09820d9d-3c12-45f3-bc62-97b53f8a7efe",
-      "alias": "review profile config",
-      "config": {
-        "update.profile.on.first.login": "missing"
-      }
-    }
-  ],
-  "requiredActions": [
-    {
-      "alias": "CONFIGURE_TOTP",
-      "name": "Configure OTP",
-      "providerId": "CONFIGURE_TOTP",
-      "enabled": true,
-      "defaultAction": false,
-      "priority": 10,
-      "config": {}
-    },
-    {
-      "alias": "terms_and_conditions",
-      "name": "Terms and Conditions",
-      "providerId": "terms_and_conditions",
-      "enabled": false,
-      "defaultAction": false,
-      "priority": 20,
-      "config": {}
-    },
-    {
-      "alias": "UPDATE_PASSWORD",
-      "name": "Update Password",
-      "providerId": "UPDATE_PASSWORD",
-      "enabled": true,
-      "defaultAction": false,
-      "priority": 30,
-      "config": {}
-    },
-    {
-      "alias": "UPDATE_PROFILE",
-      "name": "Update Profile",
-      "providerId": "UPDATE_PROFILE",
-      "enabled": true,
-      "defaultAction": false,
-      "priority": 40,
-      "config": {}
-    },
-    {
-      "alias": "VERIFY_EMAIL",
-      "name": "Verify Email",
-      "providerId": "VERIFY_EMAIL",
-      "enabled": true,
-      "defaultAction": false,
-      "priority": 50,
-      "config": {}
-    },
-    {
-      "alias": "delete_account",
-      "name": "Delete Account",
-      "providerId": "delete_account",
-      "enabled": false,
-      "defaultAction": false,
-      "priority": 60,
-      "config": {}
-    },
-    {
-      "alias": "update_user_locale",
-      "name": "Update User Locale",
-      "providerId": "update_user_locale",
-      "enabled": true,
-      "defaultAction": false,
-      "priority": 1000,
-      "config": {}
-    }
-  ],
-  "browserFlow": "browser",
-  "registrationFlow": "registration",
-  "directGrantFlow": "direct grant",
-  "resetCredentialsFlow": "reset credentials",
-  "clientAuthenticationFlow": "clients",
-  "dockerAuthenticationFlow": "docker auth",
-  "attributes": {
-    "cibaBackchannelTokenDeliveryMode": "poll",
-    "cibaExpiresIn": "120",
-    "cibaAuthRequestedUserHint": "login_hint",
-    "oauth2DeviceCodeLifespan": "600",
-    "clientOfflineSessionMaxLifespan": "0",
-    "oauth2DevicePollingInterval": "5",
-    "clientSessionIdleTimeout": "0",
-    "parRequestUriLifespan": "60",
-    "clientSessionMaxLifespan": "0",
-    "clientOfflineSessionIdleTimeout": "0",
-    "cibaInterval": "5"
-  },
-  "keycloakVersion": "16.1.1",
-  "userManagedAccessAllowed": false,
-  "clientProfiles": {
-    "profiles": []
-  },
-  "clientPolicies": {
-    "policies": []
-  }
-}
diff --git a/conf/keycloak/run-keycloak.sh b/conf/keycloak/run-keycloak.sh
index effb37f91b8..ddc5108bee4 100755
--- a/conf/keycloak/run-keycloak.sh
+++ b/conf/keycloak/run-keycloak.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-DOCKER_IMAGE="jboss/keycloak:16.1.1"
+DOCKER_IMAGE="quay.io/keycloak/keycloak:21.0"
 KEYCLOAK_USER="kcadmin"
 KEYCLOAK_PASSWORD="kcpassword"
 KEYCLOAK_PORT=8090
@@ -11,7 +11,7 @@ if [ ! "$(docker ps -q -f name=^/keycloak$)" ]; then
     docker start keycloak
     echo "INFO - Keycloak container restarted"
   else
-    docker run -d --name keycloak -p $KEYCLOAK_PORT:8080 -e KEYCLOAK_USER=$KEYCLOAK_USER -e KEYCLOAK_PASSWORD=$KEYCLOAK_PASSWORD -e KEYCLOAK_IMPORT=/tmp/oidc-realm.json -v "$(pwd)"/oidc-realm.json:/tmp/oidc-realm.json $DOCKER_IMAGE
+    docker run -d --name keycloak -p $KEYCLOAK_PORT:8080 -e KEYCLOAK_USER=$KEYCLOAK_USER -e KEYCLOAK_PASSWORD=$KEYCLOAK_PASSWORD -e KEYCLOAK_IMPORT=/tmp/test-realm.json -v "$(pwd)"/test-realm.json:/tmp/test-realm.json $DOCKER_IMAGE
     echo "INFO - Keycloak container created and running"
   fi
 else
diff --git a/conf/keycloak/test-realm.json b/conf/keycloak/test-realm.json
new file mode 100644
index 00000000000..efe71cc5d29
--- /dev/null
+++ b/conf/keycloak/test-realm.json
@@ -0,0 +1,1939 @@
+{
+  "id" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983",
+  "realm" : "test",
+  "displayName" : "",
+  "displayNameHtml" : "",
+  "notBefore" : 0,
+  "defaultSignatureAlgorithm" : "RS256",
+  "revokeRefreshToken" : false,
+  "refreshTokenMaxReuse" : 0,
+  "accessTokenLifespan" : 300,
+  "accessTokenLifespanForImplicitFlow" : 900,
+  "ssoSessionIdleTimeout" : 1800,
+  "ssoSessionMaxLifespan" : 36000,
+  "ssoSessionIdleTimeoutRememberMe" : 0,
+  "ssoSessionMaxLifespanRememberMe" : 0,
+  "offlineSessionIdleTimeout" : 2592000,
+  "offlineSessionMaxLifespanEnabled" : false,
+  "offlineSessionMaxLifespan" : 5184000,
+  "clientSessionIdleTimeout" : 0,
+  "clientSessionMaxLifespan" : 0,
+  "clientOfflineSessionIdleTimeout" : 0,
+  "clientOfflineSessionMaxLifespan" : 0,
+  "accessCodeLifespan" : 60,
+  "accessCodeLifespanUserAction" : 300,
+  "accessCodeLifespanLogin" : 1800,
+  "actionTokenGeneratedByAdminLifespan" : 43200,
+  "actionTokenGeneratedByUserLifespan" : 300,
+  "oauth2DeviceCodeLifespan" : 600,
+  "oauth2DevicePollingInterval" : 5,
+  "enabled" : true,
+  "sslRequired" : "none",
+  "registrationAllowed" : false,
+  "registrationEmailAsUsername" : false,
+  "rememberMe" : false,
+  "verifyEmail" : false,
+  "loginWithEmailAllowed" : true,
+  "duplicateEmailsAllowed" : false,
+  "resetPasswordAllowed" : false,
+  "editUsernameAllowed" : false,
+  "bruteForceProtected" : false,
+  "permanentLockout" : false,
+  "maxFailureWaitSeconds" : 900,
+  "minimumQuickLoginWaitSeconds" : 60,
+  "waitIncrementSeconds" : 60,
+  "quickLoginCheckMilliSeconds" : 1000,
+  "maxDeltaTimeSeconds" : 43200,
+  "failureFactor" : 30,
+  "roles" : {
+    "realm" : [ {
+      "id" : "075daee1-5ab2-44b5-adbf-fa49a3da8305",
+      "name" : "uma_authorization",
+      "description" : "${role_uma_authorization}",
+      "composite" : false,
+      "clientRole" : false,
+      "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983",
+      "attributes" : { }
+    }, {
+      "id" : "b4ff9091-ddf9-4536-b175-8cfa3e331d71",
+      "name" : "default-roles-test",
+      "description" : "${role_default-roles}",
+      "composite" : true,
+      "composites" : {
+        "realm" : [ "offline_access", "uma_authorization" ],
+        "client" : {
+          "account" : [ "view-profile", "manage-account" ]
+        }
+      },
+      "clientRole" : false,
+      "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983",
+      "attributes" : { }
+    }, {
+      "id" : "e6d31555-6be6-4dee-bc6a-40a53108e4c2",
+      "name" : "offline_access",
+      "description" : "${role_offline-access}",
+      "composite" : false,
+      "clientRole" : false,
+      "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983",
+      "attributes" : { }
+    } ],
+    "client" : {
+      "realm-management" : [ {
+        "id" : "1955bd12-5f86-4a74-b130-d68a8ef6f0ee",
+        "name" : "impersonation",
+        "description" : "${role_impersonation}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "1109c350-9ab1-426c-9876-ef67d4310f35",
+        "name" : "view-authorization",
+        "description" : "${role_view-authorization}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "980c3fd3-1ae3-4b8f-9a00-d764c939035f",
+        "name" : "query-users",
+        "description" : "${role_query-users}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "5363e601-0f9d-4633-a8c8-28cb0f859b7b",
+        "name" : "query-groups",
+        "description" : "${role_query-groups}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "59aa7992-ad78-48db-868a-25d6e1d7db50",
+        "name" : "realm-admin",
+        "description" : "${role_realm-admin}",
+        "composite" : true,
+        "composites" : {
+          "client" : {
+            "realm-management" : [ "impersonation", "view-authorization", "query-users", "query-groups", "manage-clients", "manage-realm", "view-identity-providers", "query-realms", "manage-authorization", "manage-identity-providers", "manage-users", "view-users", "view-realm", "create-client", "view-clients", "manage-events", "query-clients", "view-events" ]
+          }
+        },
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "112f53c2-897d-4c01-81db-b8dc10c5b995",
+        "name" : "manage-clients",
+        "description" : "${role_manage-clients}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "c7f57bbd-ef32-4a64-9888-7b8abd90777a",
+        "name" : "manage-realm",
+        "description" : "${role_manage-realm}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "8885dac8-0af3-45af-94ce-eff5e801bb80",
+        "name" : "view-identity-providers",
+        "description" : "${role_view-identity-providers}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "2673346c-b0ef-4e01-8a90-be03866093af",
+        "name" : "manage-authorization",
+        "description" : "${role_manage-authorization}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "b7182885-9e57-445f-8dae-17c16eb31b5d",
+        "name" : "manage-identity-providers",
+        "description" : "${role_manage-identity-providers}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "ba7bfe0c-cb07-4a47-b92c-b8132b57e181",
+        "name" : "manage-users",
+        "description" : "${role_manage-users}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "13a8f0fc-647d-4bfe-b525-73956898e550",
+        "name" : "query-realms",
+        "description" : "${role_query-realms}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "ef4c57dc-78c2-4f9a-8d2b-0e97d46fc842",
+        "name" : "view-realm",
+        "description" : "${role_view-realm}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "2875da34-006c-4b7f-bfc8-9ae8e46af3a2",
+        "name" : "view-users",
+        "description" : "${role_view-users}",
+        "composite" : true,
+        "composites" : {
+          "client" : {
+            "realm-management" : [ "query-users", "query-groups" ]
+          }
+        },
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "c8c8f7dc-876b-4263-806f-3329f7cd5fd3",
+        "name" : "create-client",
+        "description" : "${role_create-client}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "21b84f90-5a9a-4845-a7ba-bbd98ac0fcc4",
+        "name" : "view-clients",
+        "description" : "${role_view-clients}",
+        "composite" : true,
+        "composites" : {
+          "client" : {
+            "realm-management" : [ "query-clients" ]
+          }
+        },
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "6fd64c94-d663-4501-ad77-0dcf8887d434",
+        "name" : "manage-events",
+        "description" : "${role_manage-events}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "b321927a-023c-4d2a-99ad-24baf7ff6d83",
+        "name" : "query-clients",
+        "description" : "${role_query-clients}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      }, {
+        "id" : "2fc21160-78de-457b-8594-e5c76cde1d5e",
+        "name" : "view-events",
+        "description" : "${role_view-events}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+        "attributes" : { }
+      } ],
+      "test" : [ ],
+      "security-admin-console" : [ ],
+      "admin-cli" : [ ],
+      "account-console" : [ ],
+      "broker" : [ {
+        "id" : "07ee59b5-dca6-48fb-83d4-2994ef02850e",
+        "name" : "read-token",
+        "description" : "${role_read-token}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "b57d62bb-77ff-42bd-b8ff-381c7288f327",
+        "attributes" : { }
+      } ],
+      "account" : [ {
+        "id" : "17d2f811-7bdf-4c73-83b4-1037001797b8",
+        "name" : "view-applications",
+        "description" : "${role_view-applications}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      }, {
+        "id" : "d1ff44f9-419e-42fd-98e8-1add1169a972",
+        "name" : "delete-account",
+        "description" : "${role_delete-account}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      }, {
+        "id" : "14c23a18-ae2d-43c9-b0c0-aaf6e0c7f5b0",
+        "name" : "manage-account-links",
+        "description" : "${role_manage-account-links}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      }, {
+        "id" : "6fbe58af-d2fe-4d66-95fe-a2e8a818cb55",
+        "name" : "view-profile",
+        "description" : "${role_view-profile}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      }, {
+        "id" : "bdfd02bc-6f6a-47d2-82bc-0ca52d78ff48",
+        "name" : "manage-consent",
+        "description" : "${role_manage-consent}",
+        "composite" : true,
+        "composites" : {
+          "client" : {
+            "account" : [ "view-consent" ]
+          }
+        },
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      }, {
+        "id" : "782f3b0c-a17b-4a87-988b-1a711401f3b0",
+        "name" : "manage-account",
+        "description" : "${role_manage-account}",
+        "composite" : true,
+        "composites" : {
+          "client" : {
+            "account" : [ "manage-account-links" ]
+          }
+        },
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      }, {
+        "id" : "8a3bfe15-66d9-4f3d-83ac-801d682d42b0",
+        "name" : "view-consent",
+        "description" : "${role_view-consent}",
+        "composite" : false,
+        "clientRole" : true,
+        "containerId" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+        "attributes" : { }
+      } ]
+    }
+  },
+  "groups" : [ {
+    "id" : "d46f94c2-3b47-4288-b937-9cf918e54f0a",
+    "name" : "admins",
+    "path" : "/admins",
+    "attributes" : { },
+    "realmRoles" : [ ],
+    "clientRoles" : { },
+    "subGroups" : [ ]
+  }, {
+    "id" : "e992ce15-baac-48a0-8834-06f6fcf6c05b",
+    "name" : "curators",
+    "path" : "/curators",
+    "attributes" : { },
+    "realmRoles" : [ ],
+    "clientRoles" : { },
+    "subGroups" : [ ]
+  }, {
+    "id" : "531cf81d-a700-4336-808f-37a49709b48c",
+    "name" : "members",
+    "path" : "/members",
+    "attributes" : { },
+    "realmRoles" : [ ],
+    "clientRoles" : { },
+    "subGroups" : [ ]
+  } ],
+  "defaultRole" : {
+    "id" : "b4ff9091-ddf9-4536-b175-8cfa3e331d71",
+    "name" : "default-roles-test",
+    "description" : "${role_default-roles}",
+    "composite" : true,
+    "clientRole" : false,
+    "containerId" : "80a7e04b-a2b5-4891-a2d1-5ad4e915f983"
+  },
+  "requiredCredentials" : [ "password" ],
+  "otpPolicyType" : "totp",
+  "otpPolicyAlgorithm" : "HmacSHA1",
+  "otpPolicyInitialCounter" : 0,
+  "otpPolicyDigits" : 6,
+  "otpPolicyLookAheadWindow" : 1,
+  "otpPolicyPeriod" : 30,
+  "otpSupportedApplications" : [ "FreeOTP", "Google Authenticator" ],
+  "webAuthnPolicyRpEntityName" : "keycloak",
+  "webAuthnPolicySignatureAlgorithms" : [ "ES256" ],
+  "webAuthnPolicyRpId" : "",
+  "webAuthnPolicyAttestationConveyancePreference" : "not specified",
+  "webAuthnPolicyAuthenticatorAttachment" : "not specified",
+  "webAuthnPolicyRequireResidentKey" : "not specified",
+  "webAuthnPolicyUserVerificationRequirement" : "not specified",
+  "webAuthnPolicyCreateTimeout" : 0,
+  "webAuthnPolicyAvoidSameAuthenticatorRegister" : false,
+  "webAuthnPolicyAcceptableAaguids" : [ ],
+  "webAuthnPolicyPasswordlessRpEntityName" : "keycloak",
+  "webAuthnPolicyPasswordlessSignatureAlgorithms" : [ "ES256" ],
+  "webAuthnPolicyPasswordlessRpId" : "",
+  "webAuthnPolicyPasswordlessAttestationConveyancePreference" : "not specified",
+  "webAuthnPolicyPasswordlessAuthenticatorAttachment" : "not specified",
+  "webAuthnPolicyPasswordlessRequireResidentKey" : "not specified",
+  "webAuthnPolicyPasswordlessUserVerificationRequirement" : "not specified",
+  "webAuthnPolicyPasswordlessCreateTimeout" : 0,
+  "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister" : false,
+  "webAuthnPolicyPasswordlessAcceptableAaguids" : [ ],
+  "users" : [ {
+    "id" : "52cddd46-251c-4534-acc8-0580eeafb577",
+    "createdTimestamp" : 1684736014759,
+    "username" : "admin",
+    "enabled" : true,
+    "totp" : false,
+    "emailVerified" : true,
+    "firstName" : "Dataverse",
+    "lastName" : "Admin",
+    "email" : "dataverse-admin@mailinator.com",
+    "credentials" : [ {
+      "id" : "28f1ece7-26fb-40f1-9174-5ffce7b85c0a",
+      "type" : "password",
+      "userLabel" : "Set to \"admin\"",
+      "createdDate" : 1684736057302,
+      "secretData" : "{\"value\":\"ONI7fl6BmooVTUgwN1W3m7hsRjMAYEr2l+Fp5+7IOYw1iIntwvZ3U3W0ZBcCFJ7uhcKqF101+rueM3dZfoshPQ==\",\"salt\":\"Hj7co7zYVei7xwx8EaYP3A==\",\"additionalParameters\":{}}",
+      "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
+    } ],
+    "disableableCredentialTypes" : [ ],
+    "requiredActions" : [ ],
+    "realmRoles" : [ "default-roles-test" ],
+    "notBefore" : 0,
+    "groups" : [ "/admins" ]
+  }, {
+    "id" : "a3d8e76d-7e7b-42dc-bbd7-4258818a8a1b",
+    "createdTimestamp" : 1684755806552,
+    "username" : "affiliate",
+    "enabled" : true,
+    "totp" : false,
+    "emailVerified" : true,
+    "firstName" : "Dataverse",
+    "lastName" : "Affiliate",
+    "email" : "dataverse-affiliate@mailinator.com",
+    "credentials" : [ {
+      "id" : "31c8eb1e-b2a8-4f86-833b-7c0536cd61a1",
+      "type" : "password",
+      "userLabel" : "My password",
+      "createdDate" : 1684755821743,
+      "secretData" : "{\"value\":\"T+RQ4nvmjknj7ds8NU7782j6PJ++uCu98zNoDQjIe9IKXah+13q4EcXO9IHmi2BJ7lgT0OIzwIoac4JEQLxhjQ==\",\"salt\":\"fnRmE9WmjAp4tlvGh/bxxQ==\",\"additionalParameters\":{}}",
+      "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
+    } ],
+    "disableableCredentialTypes" : [ ],
+    "requiredActions" : [ ],
+    "realmRoles" : [ "default-roles-test" ],
+    "notBefore" : 0,
+    "groups" : [ ]
+  }, {
+    "id" : "e5531496-cfb8-498c-a902-50c98d649e79",
+    "createdTimestamp" : 1684755721064,
+    "username" : "curator",
+    "enabled" : true,
+    "totp" : false,
+    "emailVerified" : true,
+    "firstName" : "Dataverse",
+    "lastName" : "Curator",
+    "email" : "dataverse-curator@mailinator.com",
+    "credentials" : [ {
+      "id" : "664546b4-b936-45cf-a4cf-5e98b743fc7f",
+      "type" : "password",
+      "userLabel" : "My password",
+      "createdDate" : 1684755740776,
+      "secretData" : "{\"value\":\"AvVqybCNtCBVAdLEeJKresy9tc3c4BBUQvu5uHVQw4IjVagN6FpKGlDEKOrxhzdSM8skEvthOEqJkloPo1w+NQ==\",\"salt\":\"2em2DDRRlNEYsNR3xDqehw==\",\"additionalParameters\":{}}",
+      "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
+    } ],
+    "disableableCredentialTypes" : [ ],
+    "requiredActions" : [ ],
+    "realmRoles" : [ "default-roles-test" ],
+    "notBefore" : 0,
+    "groups" : [ "/curators" ]
+  }, {
+    "id" : "c0082e7e-a3e9-45e6-95e9-811a34adce9d",
+    "createdTimestamp" : 1684755585802,
+    "username" : "user",
+    "enabled" : true,
+    "totp" : false,
+    "emailVerified" : true,
+    "firstName" : "Dataverse",
+    "lastName" : "User",
+    "email" : "dataverse-user@mailinator.com",
+    "credentials" : [ {
+      "id" : "00d6d67f-2e30-4da6-a567-bec38a1886a0",
+      "type" : "password",
+      "userLabel" : "My password",
+      "createdDate" : 1684755599597,
+      "secretData" : "{\"value\":\"z991rnjznAgosi5nX962HjM8/gN5GLJTdrlvi6G9cj8470X2/oZUb4Lka6s8xImgtEloCgWiKqH0EH9G4Y3a5A==\",\"salt\":\"/Uz7w+2IqDo+fQUGqxjVHw==\",\"additionalParameters\":{}}",
+      "credentialData" : "{\"hashIterations\":27500,\"algorithm\":\"pbkdf2-sha256\",\"additionalParameters\":{}}"
+    } ],
+    "disableableCredentialTypes" : [ ],
+    "requiredActions" : [ ],
+    "realmRoles" : [ "default-roles-test" ],
+    "notBefore" : 0,
+    "groups" : [ "/members" ]
+  } ],
+  "scopeMappings" : [ {
+    "clientScope" : "offline_access",
+    "roles" : [ "offline_access" ]
+  } ],
+  "clientScopeMappings" : {
+    "account" : [ {
+      "client" : "account-console",
+      "roles" : [ "manage-account" ]
+    } ]
+  },
+  "clients" : [ {
+    "id" : "77f8127a-261e-4cd8-a77d-b74a389f7fd4",
+    "clientId" : "account",
+    "name" : "${client_account}",
+    "rootUrl" : "${authBaseUrl}",
+    "baseUrl" : "/realms/test/account/",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "redirectUris" : [ "/realms/test/account/*" ],
+    "webOrigins" : [ ],
+    "notBefore" : 0,
+    "bearerOnly" : false,
+    "consentRequired" : false,
+    "standardFlowEnabled" : true,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : false,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : true,
+    "frontchannelLogout" : false,
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "post.logout.redirect.uris" : "+"
+    },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : false,
+    "nodeReRegistrationTimeout" : 0,
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  }, {
+    "id" : "5d99f721-027c-478d-867d-61114e0a8192",
+    "clientId" : "account-console",
+    "name" : "${client_account-console}",
+    "rootUrl" : "${authBaseUrl}",
+    "baseUrl" : "/realms/test/account/",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "redirectUris" : [ "/realms/test/account/*" ],
+    "webOrigins" : [ ],
+    "notBefore" : 0,
+    "bearerOnly" : false,
+    "consentRequired" : false,
+    "standardFlowEnabled" : true,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : false,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : true,
+    "frontchannelLogout" : false,
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "post.logout.redirect.uris" : "+",
+      "pkce.code.challenge.method" : "S256"
+    },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : false,
+    "nodeReRegistrationTimeout" : 0,
+    "protocolMappers" : [ {
+      "id" : "e181a0ce-9a04-4468-a38a-aaef9f78f989",
+      "name" : "audience resolve",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-audience-resolve-mapper",
+      "consentRequired" : false,
+      "config" : { }
+    } ],
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  }, {
+    "id" : "5eccc178-121e-4d0f-bcb2-04ae3c2e52ed",
+    "clientId" : "admin-cli",
+    "name" : "${client_admin-cli}",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "redirectUris" : [ ],
+    "webOrigins" : [ ],
+    "notBefore" : 0,
+    "bearerOnly" : false,
+    "consentRequired" : false,
+    "standardFlowEnabled" : false,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : true,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : true,
+    "frontchannelLogout" : false,
+    "protocol" : "openid-connect",
+    "attributes" : { },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : false,
+    "nodeReRegistrationTimeout" : 0,
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  }, {
+    "id" : "b57d62bb-77ff-42bd-b8ff-381c7288f327",
+    "clientId" : "broker",
+    "name" : "${client_broker}",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "redirectUris" : [ ],
+    "webOrigins" : [ ],
+    "notBefore" : 0,
+    "bearerOnly" : true,
+    "consentRequired" : false,
+    "standardFlowEnabled" : true,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : false,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : false,
+    "frontchannelLogout" : false,
+    "protocol" : "openid-connect",
+    "attributes" : { },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : false,
+    "nodeReRegistrationTimeout" : 0,
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  }, {
+    "id" : "dada0ae8-ee9f-415a-9685-42da7c563660",
+    "clientId" : "realm-management",
+    "name" : "${client_realm-management}",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "redirectUris" : [ ],
+    "webOrigins" : [ ],
+    "notBefore" : 0,
+    "bearerOnly" : true,
+    "consentRequired" : false,
+    "standardFlowEnabled" : true,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : false,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : false,
+    "frontchannelLogout" : false,
+    "protocol" : "openid-connect",
+    "attributes" : { },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : false,
+    "nodeReRegistrationTimeout" : 0,
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  }, {
+    "id" : "bf7cf550-3875-4f97-9878-b2419a854058",
+    "clientId" : "security-admin-console",
+    "name" : "${client_security-admin-console}",
+    "rootUrl" : "${authAdminUrl}",
+    "baseUrl" : "/admin/test/console/",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "redirectUris" : [ "/admin/test/console/*" ],
+    "webOrigins" : [ "+" ],
+    "notBefore" : 0,
+    "bearerOnly" : false,
+    "consentRequired" : false,
+    "standardFlowEnabled" : true,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : false,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : true,
+    "frontchannelLogout" : false,
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "post.logout.redirect.uris" : "+",
+      "pkce.code.challenge.method" : "S256"
+    },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : false,
+    "nodeReRegistrationTimeout" : 0,
+    "protocolMappers" : [ {
+      "id" : "ff845e16-e200-4894-ab51-37d8b9f2a445",
+      "name" : "locale",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "locale",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "locale",
+        "jsonType.label" : "String"
+      }
+    } ],
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  }, {
+    "id" : "9c27faa8-4b8d-4ad9-9cd1-880032ef06aa",
+    "clientId" : "test",
+    "name" : "A Test Client",
+    "description" : "Use for hacking and testing away a confidential client",
+    "rootUrl" : "",
+    "adminUrl" : "",
+    "baseUrl" : "",
+    "surrogateAuthRequired" : false,
+    "enabled" : true,
+    "alwaysDisplayInConsole" : false,
+    "clientAuthenticatorType" : "client-secret",
+    "secret" : "94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8",
+    "redirectUris" : [ "*" ],
+    "webOrigins" : [ ],
+    "notBefore" : 0,
+    "bearerOnly" : false,
+    "consentRequired" : false,
+    "standardFlowEnabled" : true,
+    "implicitFlowEnabled" : false,
+    "directAccessGrantsEnabled" : true,
+    "serviceAccountsEnabled" : false,
+    "publicClient" : false,
+    "frontchannelLogout" : true,
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "oidc.ciba.grant.enabled" : "false",
+      "client.secret.creation.time" : "1684735831",
+      "backchannel.logout.session.required" : "true",
+      "display.on.consent.screen" : "false",
+      "oauth2.device.authorization.grant.enabled" : "false",
+      "backchannel.logout.revoke.offline.tokens" : "false"
+    },
+    "authenticationFlowBindingOverrides" : { },
+    "fullScopeAllowed" : true,
+    "nodeReRegistrationTimeout" : -1,
+    "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ],
+    "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ]
+  } ],
+  "clientScopes" : [ {
+    "id" : "72f29e57-92fa-437b-828c-2b9d6fe56192",
+    "name" : "address",
+    "description" : "OpenID Connect built-in scope: address",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "true",
+      "display.on.consent.screen" : "true",
+      "consent.screen.text" : "${addressScopeConsentText}"
+    },
+    "protocolMappers" : [ {
+      "id" : "59581aea-70d6-4ee8-bec2-1fea5fc497ae",
+      "name" : "address",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-address-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "user.attribute.formatted" : "formatted",
+        "user.attribute.country" : "country",
+        "user.attribute.postal_code" : "postal_code",
+        "userinfo.token.claim" : "true",
+        "user.attribute.street" : "street",
+        "id.token.claim" : "true",
+        "user.attribute.region" : "region",
+        "access.token.claim" : "true",
+        "user.attribute.locality" : "locality"
+      }
+    } ]
+  }, {
+    "id" : "f515ec81-3c1b-4d4d-b7a2-e7e8d47b6447",
+    "name" : "roles",
+    "description" : "OpenID Connect scope for add user roles to the access token",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "false",
+      "display.on.consent.screen" : "true",
+      "consent.screen.text" : "${rolesScopeConsentText}"
+    },
+    "protocolMappers" : [ {
+      "id" : "26d299a8-69e2-4864-9595-17a5b417fc61",
+      "name" : "realm roles",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-realm-role-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "user.attribute" : "foo",
+        "access.token.claim" : "true",
+        "claim.name" : "realm_access.roles",
+        "jsonType.label" : "String",
+        "multivalued" : "true"
+      }
+    }, {
+      "id" : "d2998083-a8db-4f4e-9aaa-9cad68d65b97",
+      "name" : "audience resolve",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-audience-resolve-mapper",
+      "consentRequired" : false,
+      "config" : { }
+    }, {
+      "id" : "7a4cb2e5-07a0-4c16-a024-71df7ddd6868",
+      "name" : "client roles",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-client-role-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "user.attribute" : "foo",
+        "access.token.claim" : "true",
+        "claim.name" : "resource_access.${client_id}.roles",
+        "jsonType.label" : "String",
+        "multivalued" : "true"
+      }
+    } ]
+  }, {
+    "id" : "8f1eafef-92d6-434e-b9ec-6edec1fddd0a",
+    "name" : "offline_access",
+    "description" : "OpenID Connect built-in scope: offline_access",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "consent.screen.text" : "${offlineAccessScopeConsentText}",
+      "display.on.consent.screen" : "true"
+    }
+  }, {
+    "id" : "c03095aa-b656-447a-9767-0763c2ccb070",
+    "name" : "acr",
+    "description" : "OpenID Connect scope for add acr (authentication context class reference) to the token",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "false",
+      "display.on.consent.screen" : "false"
+    },
+    "protocolMappers" : [ {
+      "id" : "948b230c-56d0-4000-937c-841cd395d3f9",
+      "name" : "acr loa level",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-acr-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "id.token.claim" : "true",
+        "access.token.claim" : "true"
+      }
+    } ]
+  }, {
+    "id" : "cdf35f63-8ec7-41a0-ae12-f05d415818cc",
+    "name" : "phone",
+    "description" : "OpenID Connect built-in scope: phone",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "true",
+      "display.on.consent.screen" : "true",
+      "consent.screen.text" : "${phoneScopeConsentText}"
+    },
+    "protocolMappers" : [ {
+      "id" : "ba4348ff-90b1-4e09-89a8-e5c08b04d3d1",
+      "name" : "phone number",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "phoneNumber",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "phone_number",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "e6cceae5-8392-4348-b302-f610ece6056e",
+      "name" : "phone number verified",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "phoneNumberVerified",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "phone_number_verified",
+        "jsonType.label" : "boolean"
+      }
+    } ]
+  }, {
+    "id" : "4318001c-2970-41d3-91b9-e31c08569872",
+    "name" : "email",
+    "description" : "OpenID Connect built-in scope: email",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "true",
+      "display.on.consent.screen" : "true",
+      "consent.screen.text" : "${emailScopeConsentText}"
+    },
+    "protocolMappers" : [ {
+      "id" : "406d02a6-866a-4962-8838-e8c58ada1505",
+      "name" : "email",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-property-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "email",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "email",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "33baabc1-9bf2-42e4-8b8e-a53c13f0b744",
+      "name" : "email verified",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-property-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "emailVerified",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "email_verified",
+        "jsonType.label" : "boolean"
+      }
+    } ]
+  }, {
+    "id" : "5277a84f-d727-4c64-8432-d513127beee1",
+    "name" : "profile",
+    "description" : "OpenID Connect built-in scope: profile",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "true",
+      "display.on.consent.screen" : "true",
+      "consent.screen.text" : "${profileScopeConsentText}"
+    },
+    "protocolMappers" : [ {
+      "id" : "0a609875-2678-4056-93ef-dd5c03e6059d",
+      "name" : "given name",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-property-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "firstName",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "given_name",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "7c510d18-07ee-4b78-8acd-24b777d11b3c",
+      "name" : "website",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "website",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "website",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "0bb6d0ea-195f-49e8-918c-c419a26a661c",
+      "name" : "username",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-property-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "username",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "preferred_username",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "5f1e644c-1acf-440c-b1a6-b5f65bcebfd9",
+      "name" : "profile",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "profile",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "profile",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "c710bdb2-6cfd-4f60-9c4e-730188fc62f7",
+      "name" : "family name",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-property-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "lastName",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "family_name",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "012d5038-0e13-42ba-9df7-2487c8e2eead",
+      "name" : "nickname",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "nickname",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "nickname",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "21590b19-517d-4b6d-92f6-d4f71238677e",
+      "name" : "updated at",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "updatedAt",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "updated_at",
+        "jsonType.label" : "long"
+      }
+    }, {
+      "id" : "e4cddca7-1360-42f3-9854-da6cbe00c71e",
+      "name" : "birthdate",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "birthdate",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "birthdate",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "afee328f-c64c-43e6-80d0-be2721c2ed0e",
+      "name" : "locale",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "locale",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "locale",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "780a1e2c-5b63-46f4-a5bf-dc3fd8ce0cbb",
+      "name" : "full name",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-full-name-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "userinfo.token.claim" : "true"
+      }
+    }, {
+      "id" : "aeebffff-f776-427e-83ed-064707ffce57",
+      "name" : "zoneinfo",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "zoneinfo",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "zoneinfo",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "b3e840a2-1794-4da1-bf69-31905cbff0d6",
+      "name" : "middle name",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "middleName",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "middle_name",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "0607e0e4-4f7f-4214-996d-3599772ce1c7",
+      "name" : "picture",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "picture",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "picture",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "426a609b-4e28-4132-af0d-13297b8cb63a",
+      "name" : "gender",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-attribute-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "gender",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "gender",
+        "jsonType.label" : "String"
+      }
+    } ]
+  }, {
+    "id" : "a1ebde82-ce21-438f-a3ad-261d3eeb1c01",
+    "name" : "role_list",
+    "description" : "SAML role list",
+    "protocol" : "saml",
+    "attributes" : {
+      "consent.screen.text" : "${samlRoleListScopeConsentText}",
+      "display.on.consent.screen" : "true"
+    },
+    "protocolMappers" : [ {
+      "id" : "64653ac7-7ffc-4f7c-a589-03e3b68bbd25",
+      "name" : "role list",
+      "protocol" : "saml",
+      "protocolMapper" : "saml-role-list-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "single" : "false",
+        "attribute.nameformat" : "Basic",
+        "attribute.name" : "Role"
+      }
+    } ]
+  }, {
+    "id" : "aeb5b852-dfec-4e67-9d9e-104abe9b3bf2",
+    "name" : "web-origins",
+    "description" : "OpenID Connect scope for add allowed web origins to the access token",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "false",
+      "display.on.consent.screen" : "false",
+      "consent.screen.text" : ""
+    },
+    "protocolMappers" : [ {
+      "id" : "e2fa8437-a0f1-46fc-af9c-c40fc09cd6a1",
+      "name" : "allowed web origins",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-allowed-origins-mapper",
+      "consentRequired" : false,
+      "config" : { }
+    } ]
+  }, {
+    "id" : "4fecd0d7-d4ad-457e-90f2-c7202bf01ff5",
+    "name" : "microprofile-jwt",
+    "description" : "Microprofile - JWT built-in scope",
+    "protocol" : "openid-connect",
+    "attributes" : {
+      "include.in.token.scope" : "true",
+      "display.on.consent.screen" : "false"
+    },
+    "protocolMappers" : [ {
+      "id" : "a9536634-a9f6-4ed5-a8e7-8379d3b002ca",
+      "name" : "upn",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-property-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "userinfo.token.claim" : "true",
+        "user.attribute" : "username",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "upn",
+        "jsonType.label" : "String"
+      }
+    }, {
+      "id" : "2ce1a702-9458-4926-9b8a-f82c07215755",
+      "name" : "groups",
+      "protocol" : "openid-connect",
+      "protocolMapper" : "oidc-usermodel-realm-role-mapper",
+      "consentRequired" : false,
+      "config" : {
+        "multivalued" : "true",
+        "user.attribute" : "foo",
+        "id.token.claim" : "true",
+        "access.token.claim" : "true",
+        "claim.name" : "groups",
+        "jsonType.label" : "String"
+      }
+    } ]
+  } ],
+  "defaultDefaultClientScopes" : [ "role_list", "profile", "email", "roles", "web-origins", "acr" ],
+  "defaultOptionalClientScopes" : [ "offline_access", "address", "phone", "microprofile-jwt" ],
+  "browserSecurityHeaders" : {
+    "contentSecurityPolicyReportOnly" : "",
+    "xContentTypeOptions" : "nosniff",
+    "xRobotsTag" : "none",
+    "xFrameOptions" : "SAMEORIGIN",
+    "contentSecurityPolicy" : "frame-src 'self'; frame-ancestors 'self'; object-src 'none';",
+    "xXSSProtection" : "1; mode=block",
+    "strictTransportSecurity" : "max-age=31536000; includeSubDomains"
+  },
+  "smtpServer" : { },
+  "eventsEnabled" : false,
+  "eventsListeners" : [ "jboss-logging" ],
+  "enabledEventTypes" : [ ],
+  "adminEventsEnabled" : false,
+  "adminEventsDetailsEnabled" : false,
+  "identityProviders" : [ ],
+  "identityProviderMappers" : [ ],
+  "components" : {
+    "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy" : [ {
+      "id" : "8115796f-8f1f-4d6a-88f8-ca2938451260",
+      "name" : "Allowed Client Scopes",
+      "providerId" : "allowed-client-templates",
+      "subType" : "authenticated",
+      "subComponents" : { },
+      "config" : {
+        "allow-default-scopes" : [ "true" ]
+      }
+    }, {
+      "id" : "044bd055-714d-478e-aa93-303d2161c427",
+      "name" : "Allowed Protocol Mapper Types",
+      "providerId" : "allowed-protocol-mappers",
+      "subType" : "authenticated",
+      "subComponents" : { },
+      "config" : {
+        "allowed-protocol-mapper-types" : [ "saml-user-property-mapper", "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-role-list-mapper", "saml-user-attribute-mapper", "oidc-usermodel-property-mapper", "oidc-usermodel-attribute-mapper", "oidc-full-name-mapper" ]
+      }
+    }, {
+      "id" : "be465734-3b0f-4370-a144-73db756e23f8",
+      "name" : "Allowed Protocol Mapper Types",
+      "providerId" : "allowed-protocol-mappers",
+      "subType" : "anonymous",
+      "subComponents" : { },
+      "config" : {
+        "allowed-protocol-mapper-types" : [ "oidc-usermodel-attribute-mapper", "saml-user-property-mapper", "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-user-attribute-mapper", "oidc-full-name-mapper", "oidc-usermodel-property-mapper", "saml-role-list-mapper" ]
+      }
+    }, {
+      "id" : "42a2f64d-ac9e-4221-9cf6-40ff8c868629",
+      "name" : "Trusted Hosts",
+      "providerId" : "trusted-hosts",
+      "subType" : "anonymous",
+      "subComponents" : { },
+      "config" : {
+        "host-sending-registration-request-must-match" : [ "true" ],
+        "client-uris-must-match" : [ "true" ]
+      }
+    }, {
+      "id" : "7ca08915-6c33-454c-88f2-20e1d6553b26",
+      "name" : "Max Clients Limit",
+      "providerId" : "max-clients",
+      "subType" : "anonymous",
+      "subComponents" : { },
+      "config" : {
+        "max-clients" : [ "200" ]
+      }
+    }, {
+      "id" : "f01f2b6f-3f01-4d01-b2f4-70577c6f599c",
+      "name" : "Allowed Client Scopes",
+      "providerId" : "allowed-client-templates",
+      "subType" : "anonymous",
+      "subComponents" : { },
+      "config" : {
+        "allow-default-scopes" : [ "true" ]
+      }
+    }, {
+      "id" : "516d7f21-f21a-4690-831e-36ad313093b2",
+      "name" : "Consent Required",
+      "providerId" : "consent-required",
+      "subType" : "anonymous",
+      "subComponents" : { },
+      "config" : { }
+    }, {
+      "id" : "c79df6a0-d4d8-4866-b9e6-8ddb5d1bd38e",
+      "name" : "Full Scope Disabled",
+      "providerId" : "scope",
+      "subType" : "anonymous",
+      "subComponents" : { },
+      "config" : { }
+    } ],
+    "org.keycloak.userprofile.UserProfileProvider" : [ {
+      "id" : "cf47a21f-c8fb-42f2-9bff-feca967db183",
+      "providerId" : "declarative-user-profile",
+      "subComponents" : { },
+      "config" : { }
+    } ],
+    "org.keycloak.keys.KeyProvider" : [ {
+      "id" : "6b4a2281-a9e8-43ab-aee7-190ae91b2842",
+      "name" : "aes-generated",
+      "providerId" : "aes-generated",
+      "subComponents" : { },
+      "config" : {
+        "kid" : [ "47b9c2c2-32dc-4317-bd8b-1c4e5bb740ca" ],
+        "secret" : [ "9VWsVSqbj5zWa8Mq-rRzOw" ],
+        "priority" : [ "100" ]
+      }
+    }, {
+      "id" : "68e2d2b0-4976-480f-ab76-f84a17686b05",
+      "name" : "rsa-enc-generated",
+      "providerId" : "rsa-enc-generated",
+      "subComponents" : { },
+      "config" : {
+        "privateKey" : [ "MIIEpQIBAAKCAQEAwuIcVVJDncorsQcFef4M/J9dsaNNmwEv/+4pCSZuco7IlA9uCfvwjYgfwQlWoCHCc7JFEtUOXhpLNR0SJ9w2eCC9A/0horjLmiVGU5sGACGrAxSgipt399k83mtkPBTikT1BXumPrX51ovdEPVPQSO0hIBwFn4ZDwA9P/00jNzzswyLC2UDdQrwIjm2xWjq1X82d8mL3+Yp8lF9qD1w305+XPiqCC+TUunKsuCQq5sddet+UoCDsFQyxsJi6cWJrryDvQmiDgM2wm68jn6hyzDE76J1az0wKEGqoMEwIy0juqZCyAqgsm3xA+zHpTcI3EyTwDGpMvWNJp8AWqXPNaQIDAQABAoIBAAethL1+n/6WpUBEaoHcVrq5/2+vo0+dfTyVZNKRFqtG0WOWPzOflFd1HZV7YVPuJI+uPi8ANmsnbh9YcaYg9JiTZ0hMZ++giBf0ID2hZxv995NyXnf7fkoFKghevYG+9mVPtHRmxKlKiPFWfHQjP1ACNKAD2UZdcdbzxicaIkPV/hP996mZA3xaaudggAJq7u/W67H2Q6ofGqW4TI5241d8T+6yobbvXRe4n8FKz4eK2aZv+N+zwh5JDMsJ8050+lCDsyoyakEPf+4veuPkewx4FemAiotDNcmoUQSDL26wLw8kk1uZ9JY0M88OL5pMyBuxTqy0F6BWBltq80mlefECgYEA4vZ8Agu2plXOzWASn0dyhCel3QoeUqNY8D8A+0vK9qWxUE9jMG13jAZmsL2I38SuwRN1DhJezbrn4QTuxTukxgSjLDv/pBp9UnXnCz/fg4yPTYsZ0zHqTMbwvdtfIzBHTCYyIJ+unxVYoenC0XZKSQXA3NN2zNqYpLhjStWdEZECgYEA29DznJxpDZsRUieRxFgZ+eRCjbQ9Q2A46preqMo1KOZ6bt9avxG3uM7pUC+UOeIizeRzxPSJ2SyptYPzdaNwKN3Lq+RhjHe1zYLngXb0CIQaRwNHqePxXF1sg0dTbmcxf+Co7yPG+Nd5nrQq9SQHC3tLTyL6x3VU/yAfMQqUklkCgYEAyVl8iGAV6RkE/4R04OOEv6Ng7WkVn6CUvYZXe5kw9YHnfWUAjS0AOrRPFAsBy+r0UgvN8+7uNjvTjPhQT5/rPVVN4WdVEyQA/E/m6j7/LvhbBaMbBRcqUnTHjNd6XoBtMCxOmkyvoShR2krE8AiuPHwjLoVXxsNDWhbO18wMrVECgYEAlmkICOXNzI2K8Jg62gse2yshjy0BrpSs3XtTWFPkxDPRGwSiZ5OMD10lsMSdvG3MOu5TeTWLDZvOFHJRqPFI0e3Sa7A+P4u6TwF/v8rRePJLuMO5ybo7cWRL2Bh6MlVSPZpQfjIQ+D0Y70uBCXS5jVW0VlYtG0Zh/qDQNxJyTyECgYEAuRINlZ0ag+1QTITapSatbFWd/KquGLpMjZyF4k5gVHs+4zHnnTi1YIDUInp1FJBqKD27z2byy7KFgbMBZQmsDs8i4fgzQrJHe3D4WFFHCjiClbeReejbas9bOnqhSQCiIy1Ck8vMAriAtctSA/g/qq6dQApSgcWaKvTVL2Ywa7E=" ],
+        "keyUse" : [ "ENC" ],
+        "certificate" : [ "MIIClzCCAX8CBgGIQhOIijANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMDUyMjA2MDczNloXDTMzMDUyMjA2MDkxNlowDzENMAsGA1UEAwwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMLiHFVSQ53KK7EHBXn+DPyfXbGjTZsBL//uKQkmbnKOyJQPbgn78I2IH8EJVqAhwnOyRRLVDl4aSzUdEifcNnggvQP9IaK4y5olRlObBgAhqwMUoIqbd/fZPN5rZDwU4pE9QV7pj61+daL3RD1T0EjtISAcBZ+GQ8APT/9NIzc87MMiwtlA3UK8CI5tsVo6tV/NnfJi9/mKfJRfag9cN9Oflz4qggvk1LpyrLgkKubHXXrflKAg7BUMsbCYunFia68g70Jog4DNsJuvI5+ocswxO+idWs9MChBqqDBMCMtI7qmQsgKoLJt8QPsx6U3CNxMk8AxqTL1jSafAFqlzzWkCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAIEIfjqOr2m+8s2RR8VW/nBgOgu9HtPRda4qNhGbgBkZ8NDy7TwHqlHo1ujKW5RO438pRyLJmOibWN4a/rkUsSjin6vgy4l8KpQy+7a4cQCQHyl34TmPjbtiw1jKgiOjzRQY54NVwIJNMIMc1ZyQo4u0U30/FxgUv6akXfS5O1ePD+5xKOOC/Af9AletjhQMPwVxXDwFqfQf/p+SM4Pyn4L633MESfDrH8v9FjJd0lV5ZlEI4hpPtnbi9U+CInqCy3VDNlZjsXswaDRujjg3LERfOMvCgj+Dck3FzWG7EiCwXWNEPvdMzv4w7M6KXuiPPQkST8DUWjgkjUCeLBzT3yw==" ],
+        "priority" : [ "100" ],
+        "algorithm" : [ "RSA-OAEP" ]
+      }
+    }, {
+      "id" : "728769a3-99a4-4cca-959d-28181dfee7e8",
+      "name" : "rsa-generated",
+      "providerId" : "rsa-generated",
+      "subComponents" : { },
+      "config" : {
+        "privateKey" : [ "MIIEowIBAAKCAQEAxIszQCv8bX3sKXJVtuLJV6cH/uhkzxcTEIcDe7y2Y2SFM0x2nF6wRLk8QkvIrRmelilegUIJttqZxLXMpxwUJGizehHQMrOCzNoGBZdVanoK7nNa5+FOYtlvL4GxNfwzS36sp3PnKQiGv5Q7RGuPthjLFfqTmYx/7GTDJC4vLEW5S01Vy/Xc9FE4FsT0hnm91lRWjppc9893M5QUy/TPu8udIuNV87Ko5yiIxQqcPiAQXJaN4CyGaDcYhhzzHdxVptIk2FvtxhpmNxrbtmBCx/o9/rBDQNTis8Ex6ItWC2PvC17UPvyOcZ4Fv/qO0L6JZ0mrpH95CeDU1kEP+KKZrwIDAQABAoIBAGGl6SYiVG1PyTQEXqqY/UCjt3jBnEg5ZhrpgWUKKrGyAO2uOSXSc5AJWfN0NHUwC9b+IbplhW8IJ6qQSmfiLu2x6S2mSQLPphZB4gkIGYNntCOpQ0p+aZP6BGAddt5j+VYyTvR5RKlh15S6QEHrkMB/i/LVBl0c7XeUzlEc8wnyj8DGvlmpcQzIcbWfqEZ/FciDdKGNN0M4V/r1uQiOUVZ69SWDBBwu41YwF7PYUsX83q8zn0nBeMqz0ggSf33lW4w31fox9c7EjIF01gPArE5uT+d+AwjVKHpd08LWGR9W9NSXVOPUKkzOM+PyvKGvzjMnlrm/feqowKQbL2q/GP0CgYEA/EsrvUojkFIWxHc19KJdJvqlYgLeWq6P/J7UmHgpl+S3nG6b9HH4/aM/ICDa5hxd5bmP5p2V3EuZWnyb6/QB5eipC7Ss3oM7XeS/PwvTp6NTC1fypx2zHKse3iuLeCGneRxiw15mB02ArJ/qJw/VSQK2J7RiR4+b6HYpdzQnIysCgYEAx25dTQqskQqsx/orJzuUqfNv/C0W4vqfz1eL3akFrdK+YqghXKFsDmh61JpTrTKnRLAdQeyOrhKwbNsdxSEEaeeLayKLVlimoFXGd/LZb5LQiwFcrvTzhnB+FLmFgqTnuLkpfY1woHEwSW9TpJewjbT9S6g0L2uh223nVXuLMY0CgYEA3pMOlmMGtvbEoTSuRBDNb2rmZm4zbfrcijgxRAWWZCtiFL68FU5LJLBVK2nw09sot1cabZCOuhdzxhFymRneZs73+5y8eV17DV2VnvA3HIiI5dQD/YzFDECm7ceqtiOylLUHKGZqSn0ETMaTkzxzpIKg4qxPm+RE3jMIZ+J5uJsCgYBk2iUIrtsxxgo2Xwavomu9vkPlbQ/j3QYwHn+2qqEalDZ/QbMNWvyAFMn49cpXDgSUsdM54V0OHpllkzFs3ROUUumoViHMmqw47OefBQp8Z+xaP2gVef4lAIJiDKe9t5MPUWPwADTyjgrzN/8+fw9juiFVv0wUpwOFKgEQs5diiQKBgC6RpZESc5Nl4nHrDvIl5n/zYED6BaXoLl15NhcoBudt5SIRO/RpvBW69A7aE/UK6p7WXjq4mP1ssIWz4KgATCoXUgYvn0a7Ql79r/CMce6/FvcuweED6u6bD0kdXuYhe8fR9IPmLfnnb4Cx3JOJeRZbiBSP5HOZJ7nsKibxcgPm" ],
+        "keyUse" : [ "SIG" ],
+        "certificate" : [ "MIIClzCCAX8CBgGIQhOHjjANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQDDAR0ZXN0MB4XDTIzMDUyMjA2MDczNloXDTMzMDUyMjA2MDkxNlowDzENMAsGA1UEAwwEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMSLM0Ar/G197ClyVbbiyVenB/7oZM8XExCHA3u8tmNkhTNMdpxesES5PEJLyK0ZnpYpXoFCCbbamcS1zKccFCRos3oR0DKzgszaBgWXVWp6Cu5zWufhTmLZby+BsTX8M0t+rKdz5ykIhr+UO0Rrj7YYyxX6k5mMf+xkwyQuLyxFuUtNVcv13PRROBbE9IZ5vdZUVo6aXPfPdzOUFMv0z7vLnSLjVfOyqOcoiMUKnD4gEFyWjeAshmg3GIYc8x3cVabSJNhb7cYaZjca27ZgQsf6Pf6wQ0DU4rPBMeiLVgtj7wte1D78jnGeBb/6jtC+iWdJq6R/eQng1NZBD/iima8CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAe0Bo1UpGfpOlJiVhp0XWExm8bdxFgXOU2M5XeZBsWAqBehvJkzn+tbAtlVNiIiN58XFFpH+xLZ2nJIZR5FHeCD3bYAgK72j5k45HJI95vPyslelfT/m3Np78+1iUa1U1WxN40JaowP1EeTkk5O8Pk4zTQ1Ne1usmKd+SJxI1KWN0kKuVFMmdNRb5kQKWeQvOSlWl7rd4bvHGvVnxgcPC1bshEJKRt+VpaUjpm6CKd8C3Kt7IWfIX4HTVhKZkmLn7qv6aSfwWelwZfLdaXcLXixqzqNuUk/VWbF9JT4iiag9F3mt7xryIkoRp1AEjCA82HqK72F4JCFyOhCiGrMfKJw==" ],
+        "priority" : [ "100" ]
+      }
+    }, {
+      "id" : "f30af2d2-d042-43b8-bc6d-22f6bab6934c",
+      "name" : "hmac-generated",
+      "providerId" : "hmac-generated",
+      "subComponents" : { },
+      "config" : {
+        "kid" : [ "6f0d9688-e974-42b4-9d84-8d098c51007c" ],
+        "secret" : [ "8nruwD66Revr9k21e-BHtcyvNzAMFOsstxSAB0Gdy2qe2qGRm2kYOwsPzrH9ZQSdj2041SraKo6a3SHvCyTBAQ" ],
+        "priority" : [ "100" ],
+        "algorithm" : [ "HS256" ]
+      }
+    } ]
+  },
+  "internationalizationEnabled" : false,
+  "supportedLocales" : [ ],
+  "authenticationFlows" : [ {
+    "id" : "94c65ba1-ba50-4be2-94c4-de656145eb67",
+    "alias" : "Account verification options",
+    "description" : "Method with which to verity the existing account",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "idp-email-verification",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Verify Existing Account by Re-authentication",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "3b706ddf-c4b6-498a-803c-772878bc9bc3",
+    "alias" : "Authentication Options",
+    "description" : "Authentication options.",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "basic-auth",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "basic-auth-otp",
+      "authenticatorFlow" : false,
+      "requirement" : "DISABLED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "auth-spnego",
+      "authenticatorFlow" : false,
+      "requirement" : "DISABLED",
+      "priority" : 30,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "9ea0b8f6-882c-45ad-9110-78adf5a5d233",
+    "alias" : "Browser - Conditional OTP",
+    "description" : "Flow to determine if the OTP is required for the authentication",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "conditional-user-configured",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "auth-otp-form",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "99c5ba83-b585-4601-b740-1a26670bf4e9",
+    "alias" : "Direct Grant - Conditional OTP",
+    "description" : "Flow to determine if the OTP is required for the authentication",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "conditional-user-configured",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "direct-grant-validate-otp",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "65b73dec-7dd1-4de8-b542-a023b7104afc",
+    "alias" : "First broker login - Conditional OTP",
+    "description" : "Flow to determine if the OTP is required for the authentication",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "conditional-user-configured",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "auth-otp-form",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "9a26b76f-da95-43f1-8da3-16c4a0654f07",
+    "alias" : "Handle Existing Account",
+    "description" : "Handle what to do if there is existing account with same email/username like authenticated identity provider",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "idp-confirm-link",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Account verification options",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "0a77285e-d7d5-4b6c-aa9a-3eadb5e7e3d3",
+    "alias" : "Reset - Conditional OTP",
+    "description" : "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "conditional-user-configured",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "reset-otp",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "cb6c0b3b-2f5f-4493-9d14-6130f8b58dd7",
+    "alias" : "User creation or linking",
+    "description" : "Flow for the existing/non-existing user alternatives",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticatorConfig" : "create unique user config",
+      "authenticator" : "idp-create-user-if-unique",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Handle Existing Account",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "0fd3db1b-e93d-4768-82ca-a1498ddc11d0",
+    "alias" : "Verify Existing Account by Re-authentication",
+    "description" : "Reauthentication of existing account",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "idp-username-password-form",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "CONDITIONAL",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "First broker login - Conditional OTP",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "86610e70-f9f5-4c11-8a9e-9de1770565fb",
+    "alias" : "browser",
+    "description" : "browser based authentication",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "auth-cookie",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "auth-spnego",
+      "authenticatorFlow" : false,
+      "requirement" : "DISABLED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "identity-provider-redirector",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 25,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 30,
+      "autheticatorFlow" : true,
+      "flowAlias" : "forms",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "f6aa23dd-8532-4d92-9780-3ea226481e3b",
+    "alias" : "clients",
+    "description" : "Base authentication for clients",
+    "providerId" : "client-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "client-secret",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "client-jwt",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "client-secret-jwt",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 30,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "client-x509",
+      "authenticatorFlow" : false,
+      "requirement" : "ALTERNATIVE",
+      "priority" : 40,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "4d2caf65-1703-4ddb-8890-70232e91bcd8",
+    "alias" : "direct grant",
+    "description" : "OpenID Connect Resource Owner Grant",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "direct-grant-validate-username",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "direct-grant-validate-password",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "CONDITIONAL",
+      "priority" : 30,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Direct Grant - Conditional OTP",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "eaa20c41-5334-4fb4-8c45-fb9cc71f7f74",
+    "alias" : "docker auth",
+    "description" : "Used by Docker clients to authenticate against the IDP",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "docker-http-basic-authenticator",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "b9febfb1-f0aa-4590-b782-272a4aa11575",
+    "alias" : "first broker login",
+    "description" : "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticatorConfig" : "review profile config",
+      "authenticator" : "idp-review-profile",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "User creation or linking",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "03bb6ff4-eccb-4f2f-8953-3769f78c3bf3",
+    "alias" : "forms",
+    "description" : "Username, password, otp and other auth forms.",
+    "providerId" : "basic-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "auth-username-password-form",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "CONDITIONAL",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Browser - Conditional OTP",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "38385189-246b-4ea0-ac05-d49dfe1709da",
+    "alias" : "http challenge",
+    "description" : "An authentication flow based on challenge-response HTTP Authentication Schemes",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "no-cookie-redirect",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Authentication Options",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "1022f3c2-0469-41c9-861e-918908f103df",
+    "alias" : "registration",
+    "description" : "registration flow",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "registration-page-form",
+      "authenticatorFlow" : true,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : true,
+      "flowAlias" : "registration form",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "00d36c3b-e1dc-41f8-bfd0-5f8c80ea07e8",
+    "alias" : "registration form",
+    "description" : "registration form",
+    "providerId" : "form-flow",
+    "topLevel" : false,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "registration-user-creation",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "registration-profile-action",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 40,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "registration-password-action",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 50,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "registration-recaptcha-action",
+      "authenticatorFlow" : false,
+      "requirement" : "DISABLED",
+      "priority" : 60,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "4374c16e-8c65-4168-94c2-df1ab3f3e6ad",
+    "alias" : "reset credentials",
+    "description" : "Reset credentials for a user if they forgot their password or something",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "reset-credentials-choose-user",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "reset-credential-email",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 20,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticator" : "reset-password",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 30,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    }, {
+      "authenticatorFlow" : true,
+      "requirement" : "CONDITIONAL",
+      "priority" : 40,
+      "autheticatorFlow" : true,
+      "flowAlias" : "Reset - Conditional OTP",
+      "userSetupAllowed" : false
+    } ]
+  }, {
+    "id" : "04d6ed6a-76c9-41fb-9074-bff8a80c2286",
+    "alias" : "saml ecp",
+    "description" : "SAML ECP Profile Authentication Flow",
+    "providerId" : "basic-flow",
+    "topLevel" : true,
+    "builtIn" : true,
+    "authenticationExecutions" : [ {
+      "authenticator" : "http-basic-authenticator",
+      "authenticatorFlow" : false,
+      "requirement" : "REQUIRED",
+      "priority" : 10,
+      "autheticatorFlow" : false,
+      "userSetupAllowed" : false
+    } ]
+  } ],
+  "authenticatorConfig" : [ {
+    "id" : "e7bad67d-1236-430a-a327-9194f9d1e2b0",
+    "alias" : "create unique user config",
+    "config" : {
+      "require.password.update.after.registration" : "false"
+    }
+  }, {
+    "id" : "287b5989-a927-4cf5-8067-74594ce19bc1",
+    "alias" : "review profile config",
+    "config" : {
+      "update.profile.on.first.login" : "missing"
+    }
+  } ],
+  "requiredActions" : [ {
+    "alias" : "CONFIGURE_TOTP",
+    "name" : "Configure OTP",
+    "providerId" : "CONFIGURE_TOTP",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 10,
+    "config" : { }
+  }, {
+    "alias" : "terms_and_conditions",
+    "name" : "Terms and Conditions",
+    "providerId" : "terms_and_conditions",
+    "enabled" : false,
+    "defaultAction" : false,
+    "priority" : 20,
+    "config" : { }
+  }, {
+    "alias" : "UPDATE_PASSWORD",
+    "name" : "Update Password",
+    "providerId" : "UPDATE_PASSWORD",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 30,
+    "config" : { }
+  }, {
+    "alias" : "UPDATE_PROFILE",
+    "name" : "Update Profile",
+    "providerId" : "UPDATE_PROFILE",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 40,
+    "config" : { }
+  }, {
+    "alias" : "VERIFY_EMAIL",
+    "name" : "Verify Email",
+    "providerId" : "VERIFY_EMAIL",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 50,
+    "config" : { }
+  }, {
+    "alias" : "delete_account",
+    "name" : "Delete Account",
+    "providerId" : "delete_account",
+    "enabled" : false,
+    "defaultAction" : false,
+    "priority" : 60,
+    "config" : { }
+  }, {
+    "alias" : "webauthn-register",
+    "name" : "Webauthn Register",
+    "providerId" : "webauthn-register",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 70,
+    "config" : { }
+  }, {
+    "alias" : "webauthn-register-passwordless",
+    "name" : "Webauthn Register Passwordless",
+    "providerId" : "webauthn-register-passwordless",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 80,
+    "config" : { }
+  }, {
+    "alias" : "update_user_locale",
+    "name" : "Update User Locale",
+    "providerId" : "update_user_locale",
+    "enabled" : true,
+    "defaultAction" : false,
+    "priority" : 1000,
+    "config" : { }
+  } ],
+  "browserFlow" : "browser",
+  "registrationFlow" : "registration",
+  "directGrantFlow" : "direct grant",
+  "resetCredentialsFlow" : "reset credentials",
+  "clientAuthenticationFlow" : "clients",
+  "dockerAuthenticationFlow" : "docker auth",
+  "attributes" : {
+    "cibaBackchannelTokenDeliveryMode" : "poll",
+    "cibaAuthRequestedUserHint" : "login_hint",
+    "oauth2DevicePollingInterval" : "5",
+    "clientOfflineSessionMaxLifespan" : "0",
+    "clientSessionIdleTimeout" : "0",
+    "clientOfflineSessionIdleTimeout" : "0",
+    "cibaInterval" : "5",
+    "cibaExpiresIn" : "120",
+    "oauth2DeviceCodeLifespan" : "600",
+    "parRequestUriLifespan" : "60",
+    "clientSessionMaxLifespan" : "0",
+    "frontendUrl" : ""
+  },
+  "keycloakVersion" : "19.0.3",
+  "userManagedAccessAllowed" : false,
+  "clientProfiles" : {
+    "profiles" : [ ]
+  },
+  "clientPolicies" : {
+    "policies" : [ ]
+  }
+}
\ No newline at end of file
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 30c55661a20..f00be57ea9c 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -100,8 +100,8 @@ services:
       - /mail:mode=770,size=128M,uid=1000,gid=1000
 
   dev_keycloak:
-    container_name: "dev_keycloack"
-    image: 'quay.io/keycloak/keycloak:19.0'
+    container_name: "dev_keycloak"
+    image: 'quay.io/keycloak/keycloak:21.0'
     hostname: keycloak
     environment:
       - KEYCLOAK_ADMIN=kcadmin
@@ -116,7 +116,7 @@ services:
     ports:
       - "8090:8090"
     volumes:
-      - './conf/keycloak/oidc-realm.json:/opt/keycloak/data/import/oidc-realm.json'
+      - './conf/keycloak/test-realm.json:/opt/keycloak/data/import/test-realm.json'
 
 networks:
   dataverse:

From 947ed780d8a3300c9202e62853a0b810b1039b49 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 16:10:40 +0200
Subject: [PATCH 108/922] docs(dev,oidc): update new OIDC test realm
 description

---
 .../source/developers/remote-users.rst        | 27 +++++++++++++++----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst
index d8f90e9257f..a07f104d1c8 100755
--- a/doc/sphinx-guides/source/developers/remote-users.rst
+++ b/doc/sphinx-guides/source/developers/remote-users.rst
@@ -39,7 +39,7 @@ STOP! ``oidc-keycloak-auth-provider.json`` was changed from http://localhost:809
 
 If you are working on the OpenID Connect (OIDC) user authentication flow, you do not need to connect to a remote provider (as explained in :doc:`/installation/oidc`) to test this feature. Instead, you can use the available configuration that allows you to run a test Keycloak OIDC identity management service locally through a Docker container.
 
-(Please note! The client secret (``ss6gE8mODCDfqesQaSG3gwUwZqZt547E``) is hard-coded in ``oidc-realm.json`` and ``oidc-keycloak-auth-provider.json``. Do not use this config in production! This is only for developers.)
+(Please note! The client secret (``94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8``) is hard-coded in ``test-realm.json`` and ``oidc-keycloak-auth-provider.json``. Do not use this config in production! This is only for developers.)
 
 You can find this configuration in ``conf/keycloak``. There are two options available in this directory to run a Keycloak container: bash script or docker-compose.
 
@@ -55,15 +55,32 @@ Now load the configuration defined in ``oidc-keycloak-auth-provider.json`` into
 
 You should see the new provider, called "OIDC-Keycloak", under "Other options" on the Log In page.
 
-You should be able to log into Keycloak with the following credentials:
-
-- username: kcuser
-- password: kcpassword
+You should be able to log into Keycloak with the one of the following credentials:
+
+.. list-table::
+
+  * - Username
+    - Password
+    - Group
+  * - admin
+    - admin
+    - admins
+  * - curator
+    - curator
+    - curators
+  * - user
+    - user
+    - members
+  * - affiliate
+    - affiliate
+    - \-
 
 In case you want to stop and remove the Keycloak container, just run the other available bash script:
 
 ``./rm-keycloak.sh``
 
+Note: the Keycloak admin to login at the admin console is ``kcadmin:kcpassword``
+
 ----
 
 Previous: :doc:`unf/index` | Next: :doc:`geospatial`

From 23a11718618fcc15daa3d6ddcfb18f0f49378e9d Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 16:13:47 +0200
Subject: [PATCH 109/922] test(oidc): update OIDC integration test with new
 realm

This commit enables using Keycloak 20+ with this test by replacing the test realm
and using the client to retrieve the access token on a side channel in a Keycloak
20+ compatible way. (The old one only worked for v19, this one is compatible with all)
---
 .../OIDCAuthenticationProviderFactoryIT.java  | 71 ++++++++++++-------
 1 file changed, 47 insertions(+), 24 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
index a5aa29cc083..88f70c53948 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -8,6 +8,8 @@
 import edu.harvard.iq.dataverse.api.auth.doubles.BearerTokenKeyContainerRequestTestFake;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier;
+import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception;
+import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.mocks.MockAuthenticatedUser;
@@ -17,6 +19,7 @@
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.keycloak.OAuth2Constants;
 import org.keycloak.admin.client.Keycloak;
 import org.keycloak.admin.client.KeycloakBuilder;
 import org.mockito.InjectMocks;
@@ -32,6 +35,7 @@
 import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientSecret;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assumptions.assumeFalse;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -47,18 +51,19 @@
 @JvmSetting(key = JvmSettings.OIDC_AUTH_SERVER_URL, method = "getAuthUrl")
 class OIDCAuthenticationProviderFactoryIT {
     
-    // NOTE: the following values are taken from the realm import file!
-    static final String clientId = "oidc-client";
-    static final String clientSecret = "ss6gE8mODCDfqesQaSG3gwUwZqZt547E";
-    static final String realm = "oidc-realm";
-    static final String adminUser = "kcuser";
+    static final String clientId = "test";
+    static final String clientSecret = "94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8";
+    static final String realm = "test";
+    static final String realmAdminUser = "admin";
+    static final String realmAdminPassword = "admin";
+    
+    static final String adminUser = "kcadmin";
     static final String adminPassword = "kcpassword";
-    static final String clientIdAdminCli = "admin-cli";
     
-    // The realm JSON resides in conf/keycloak/oidc-realm.json and gets avail here using <testResources> in pom.xml
+    // The realm JSON resides in conf/keycloak/test-realm.json and gets avail here using <testResources> in pom.xml
     @Container
-    static KeycloakContainer keycloakContainer = new KeycloakContainer("quay.io/keycloak/keycloak:19.0")
-        .withRealmImportFile("keycloak/oidc-realm.json")
+    static KeycloakContainer keycloakContainer = new KeycloakContainer("quay.io/keycloak/keycloak:21.0")
+        .withRealmImportFile("keycloak/test-realm.json")
         .withAdminUsername(adminUser)
         .withAdminPassword(adminPassword);
     
@@ -76,31 +81,44 @@ OIDCAuthProvider getProvider() throws Exception {
         return oidcAuthProvider;
     }
     
-    Keycloak getAdminClient() {
-        return KeycloakBuilder.builder()
+    // NOTE: This requires the "direct access grants" for the client to be enabled!
+    String getBearerTokenViaKeycloakAdminClient() throws Exception {
+        try (Keycloak keycloak = KeycloakBuilder.builder()
             .serverUrl(keycloakContainer.getAuthServerUrl())
+            .grantType(OAuth2Constants.PASSWORD)
             .realm(realm)
-            .clientId(clientIdAdminCli)
-            .username(keycloakContainer.getAdminUsername())
-            .password(keycloakContainer.getAdminPassword())
-            .build();
-    }
-    
-    String getBearerToken() throws Exception {
-        Keycloak keycloak = getAdminClient();
-        return keycloak.tokenManager().getAccessTokenString();
+            .clientId(clientId)
+            .clientSecret(clientSecret)
+            .username(realmAdminUser)
+            .password(realmAdminPassword)
+            .scope("openid")
+            .build()) {
+            return keycloak.tokenManager().getAccessTokenString();
+        }
     }
     
+    /**
+     * This basic test covers configuring an OIDC provider via MPCONFIG and being able to use it.
+     */
     @Test
     void testCreateProvider() throws Exception {
+        // given
         OIDCAuthProvider oidcAuthProvider = getProvider();
-        String token = getBearerToken();
+        String token = getBearerTokenViaKeycloakAdminClient();
         assumeFalse(token == null);
         
-        Optional<UserInfo> info = oidcAuthProvider.getUserInfo(new BearerAccessToken(token));
+        Optional<UserInfo> info = Optional.empty();
+        
+        // when
+        try {
+            info = oidcAuthProvider.getUserInfo(new BearerAccessToken(token));
+        } catch (OAuth2Exception e) {
+            System.out.println(e.getMessageBody());
+        }
         
+        //then
         assertTrue(info.isPresent());
-        assertEquals(adminUser, info.get().getPreferredUsername());
+        assertEquals(realmAdminUser, info.get().getPreferredUsername());
     }
     
     @Mock
@@ -111,6 +129,11 @@ void testCreateProvider() throws Exception {
     @InjectMocks
     BearerTokenAuthMechanism bearerTokenAuthMechanism;
     
+    /**
+     * This test covers using an OIDC provider as authorization party when accessing the Dataverse API with a
+     * Bearer Token. See {@link BearerTokenAuthMechanism}. It needs to mock the auth services to avoid adding
+     * more dependencies.
+     */
     @Test
     @JvmSetting(key = JvmSettings.FEATURE_FLAG, varArgs = "api-bearer-auth", value = "true")
     void testApiBearerAuth() throws Exception {
@@ -120,7 +143,7 @@ void testApiBearerAuth() throws Exception {
         
         // given
         // Get the access token from the remote Keycloak in the container
-        String accessToken = getBearerToken();
+        String accessToken = getBearerTokenViaKeycloakAdminClient();
         assumeFalse(accessToken == null);
         
         OIDCAuthProvider oidcAuthProvider = getProvider();

From 844339dd6adc74977a6011f58cc01dd924b002cc Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 22 May 2023 16:14:52 +0200
Subject: [PATCH 110/922] test(oidc): add third OIDC integration test for JSF
 login

Using HtmlUnit as a browser replacement, enable testing if the Authorization Code Flow
used when logging in via the UI works properly.
---
 pom.xml                                       |  6 ++
 .../OIDCAuthenticationProviderFactoryIT.java  | 78 +++++++++++++++++++
 2 files changed, 84 insertions(+)

diff --git a/pom.xml b/pom.xml
index e9a9b9dd611..2ba01e5ab61 100644
--- a/pom.xml
+++ b/pom.xml
@@ -610,6 +610,12 @@
             <version>${smallrye-mpconfig.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.htmlunit</groupId>
+            <artifactId>htmlunit</artifactId>
+            <version>3.2.0</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <testResources>
diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
index 88f70c53948..2c963e8df46 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -16,6 +16,13 @@
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
 import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
+import org.htmlunit.FailingHttpStatusCodeException;
+import org.htmlunit.WebClient;
+import org.htmlunit.WebResponse;
+import org.htmlunit.html.HtmlForm;
+import org.htmlunit.html.HtmlInput;
+import org.htmlunit.html.HtmlPage;
+import org.htmlunit.html.HtmlSubmitInput;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -28,8 +35,11 @@
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientId;
 import static edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc.OIDCAuthenticationProviderFactoryIT.clientSecret;
@@ -166,4 +176,72 @@ void testApiBearerAuth() throws Exception {
         assertNotNull(lookedUpUser);
         assertEquals(user, lookedUpUser);
     }
+    
+    /**
+     * This test covers the {@link OIDCAuthProvider#buildAuthzUrl(String, String)} and
+     * {@link OIDCAuthProvider#getUserRecord(String, String, String)} methods that are used when
+     * a user authenticates via the JSF UI. It covers enabling PKCE, which is no hard requirement
+     * by the protocol, but might be required by some provider (as seen with Microsoft Azure AD).
+     * As we don't have a real browser, we use {@link WebClient} from HtmlUnit as a replacement.
+     */
+    @Test
+    @JvmSetting(key = JvmSettings.OIDC_PKCE_ENABLED, value = "true")
+    void testAuthorizationCodeFlowWithPKCE() throws Exception {
+        // given
+        String state = "foobar";
+        String callbackUrl = "http://localhost:8080/oauth2callback.xhtml";
+        
+        OIDCAuthProvider oidcAuthProvider = getProvider();
+        String authzUrl = oidcAuthProvider.buildAuthzUrl(state, callbackUrl);
+        //System.out.println(authzUrl);
+        
+        try (WebClient webClient = new WebClient()) {
+            webClient.getOptions().setCssEnabled(false);
+            webClient.getOptions().setJavaScriptEnabled(false);
+            // We *want* to know about the redirect, as it contains the data we need!
+            webClient.getOptions().setRedirectEnabled(false);
+            
+            HtmlPage loginPage = webClient.getPage(authzUrl);
+            assumeTrue(loginPage.getTitleText().contains("Sign in to " + realm));
+            
+            HtmlForm form = loginPage.getForms().get(0);
+            HtmlInput username = form.getInputByName("username");
+            HtmlInput password = form.getInputByName("password");
+            HtmlSubmitInput submit = form.getInputByName("login");
+            
+            username.type(realmAdminUser);
+            password.type(realmAdminPassword);
+            
+            FailingHttpStatusCodeException exception = assertThrows(FailingHttpStatusCodeException.class, submit::click);
+            assertEquals(302, exception.getStatusCode());
+            
+            WebResponse response = exception.getResponse();
+            assertNotNull(response);
+            
+            String callbackLocation = response.getResponseHeaderValue("Location");
+            assertTrue(callbackLocation.startsWith(callbackUrl));
+            //System.out.println(callbackLocation);
+            
+            String queryPart = callbackLocation.trim().split("\\?")[1];
+            Map<String,String> parameters = Pattern.compile("\\s*&\\s*")
+                .splitAsStream(queryPart)
+                .map(s -> s.split("=", 2))
+                .collect(Collectors.toMap(a -> a[0], a -> a.length > 1 ? a[1]: ""));
+            //System.out.println(map);
+            assertTrue(parameters.containsKey("code"));
+            assertTrue(parameters.containsKey("state"));
+            
+            OAuth2UserRecord userRecord = oidcAuthProvider.getUserRecord(
+                parameters.get("code"),
+                parameters.get("state"),
+                callbackUrl
+            );
+            
+            assertNotNull(userRecord);
+            assertEquals(realmAdminUser, userRecord.getUsername());
+        } catch (OAuth2Exception e) {
+            System.out.println(e.getMessageBody());
+            throw e;
+        }
+    }
 }
\ No newline at end of file

From 033d8e9f58cd39d8a697354facffc0d76abf59bd Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 23 May 2023 23:12:00 +0200
Subject: [PATCH 111/922] chore(build): update many Maven plugins to latest
 available versions

This is mostly because with Maven 3.9 it starts to verify if a plugin is going to be compatible with Maven 4.
Most plugins are being updated at the moment.
---
 modules/dataverse-parent/pom.xml | 28 ++++++++++++++++++++--------
 pom.xml                          |  2 --
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 060fc22b4d2..97e83684d1a 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -174,19 +174,21 @@
         <junit.jupiter.version>5.7.0</junit.jupiter.version>
         <junit.vintage.version>${junit.jupiter.version}</junit.vintage.version>
         <mockito.version>2.28.2</mockito.version>
+        <maven-jacoco-plugin.version>0.8.10</maven-jacoco-plugin.version>
         
         <checkstyle.version>9.3</checkstyle.version>
         
         <!-- Official Maven Plugins -->
-        <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
-        <maven-jar-plugin.version>3.2.2</maven-jar-plugin.version>
+        <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
+        <maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
         <maven-war-plugin.version>3.3.2</maven-war-plugin.version>
-        <maven-dependency-plugin.version>3.2.0</maven-dependency-plugin.version>
-        <maven-install-plugin.version>3.0.0-M1</maven-install-plugin.version>
-        <maven-surefire-plugin.version>3.0.0-M5</maven-surefire-plugin.version>
-        <maven-failsafe-plugin.version>3.0.0-M5</maven-failsafe-plugin.version>
-        <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
-        <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
+        <maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version>
+        <maven-install-plugin.version>3.1.1</maven-install-plugin.version>
+        <maven-surefire-plugin.version>3.1.0</maven-surefire-plugin.version>
+        <maven-failsafe-plugin.version>3.1.0</maven-failsafe-plugin.version>
+        <maven-assembly-plugin.version>3.6.0</maven-assembly-plugin.version>
+        <maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
+        <maven-checkstyle-plugin.version>3.2.2</maven-checkstyle-plugin.version>
         
         <!-- Container related -->
         <fabric8-dmp.version>0.42.1</fabric8-dmp.version>
@@ -245,6 +247,11 @@
                     <artifactId>maven-failsafe-plugin</artifactId>
                     <version>${maven-failsafe-plugin.version}</version>
                 </plugin>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-resources-plugin</artifactId>
+                    <version>${maven-resources-plugin.version}</version>
+                </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-checkstyle-plugin</artifactId>
@@ -257,6 +264,11 @@
                         </dependency>
                     </dependencies>
                 </plugin>
+                <plugin>
+                    <groupId>org.jacoco</groupId>
+                    <artifactId>jacoco-maven-plugin</artifactId>
+                    <version>${maven-jacoco-plugin.version}</version>
+                </plugin>
                 <plugin>
                     <groupId>io.fabric8</groupId>
                     <artifactId>docker-maven-plugin</artifactId>
diff --git a/pom.xml b/pom.xml
index 2ba01e5ab61..28fc2b25af5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -26,7 +26,6 @@
         <reload4j.version>1.2.18.4</reload4j.version>
         <flyway.version>8.5.10</flyway.version>
         <jhove.version>1.20.1</jhove.version>
-        <jacoco.version>0.8.7</jacoco.version>
         <poi.version>5.2.1</poi.version>
         <tika.version>2.4.1</tika.version>
         <netcdf.version>5.5.3</netcdf.version>
@@ -704,7 +703,6 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>${jacoco.version}</version>
                 <configuration>
                     <destfile>${basedir}/target/coverage-reports/jacoco-unit.exec</destfile>
                     <datafile>${basedir}/target/coverage-reports/jacoco-unit.exec</datafile>

From f112fe46e93133e7736bfcb018623b07eebee776 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 00:05:01 +0200
Subject: [PATCH 112/922] build,test: reconfigure JaCoCo test coverage
 measuring

With the addition of integration tests we need to create a combined report
of unit tests and integration tests (for now still keeping API tests out
of the loop for normal cases).

This commit reconfigures the JaCoCo plugin with multiple executions at
certain times to execute the measurements and merge the results together.
---
 pom.xml | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 66 insertions(+), 6 deletions(-)

diff --git a/pom.xml b/pom.xml
index 28fc2b25af5..5d1523e01b8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -703,23 +703,78 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <configuration>
-                    <destfile>${basedir}/target/coverage-reports/jacoco-unit.exec</destfile>
-                    <datafile>${basedir}/target/coverage-reports/jacoco-unit.exec</datafile>
-                </configuration>
                 <executions>
                     <execution>
                         <id>jacoco-initialize</id>
                         <goals>
                             <goal>prepare-agent</goal>
                         </goals>
+                        <configuration>
+                            <destFile>${project.build.directory}/coverage-reports/jacoco-unit.exec</destFile>
+                            <propertyName>surefire.jacoco.args</propertyName>
+                        </configuration>
                     </execution>
                     <execution>
-                        <id>jacoco-site</id>
-                        <phase>package</phase>
+                        <id>jacoco-after-unit</id>
+                        <phase>test</phase>
                         <goals>
                             <goal>report</goal>
                         </goals>
+                        <configuration>
+                            <dataFile>${project.build.directory}/coverage-reports/jacoco-unit.exec</dataFile>
+                            <outputDirectory>${project.reporting.outputDirectory}/jacoco-unit-test-coverage-report</outputDirectory>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>jacoco-initialize-it</id>
+                        <phase>pre-integration-test</phase>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                        <configuration>
+                            <destFile>${project.build.directory}/coverage-reports/jacoco-integration.exec</destFile>
+                            <propertyName>failsafe.jacoco.args</propertyName>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>jacoco-after-it</id>
+                        <phase>post-integration-test</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                        <configuration>
+                            <dataFile>${project.build.directory}/coverage-reports/jacoco-integration.exec</dataFile>
+                            <outputDirectory>${project.reporting.outputDirectory}/jacoco-integration-test-coverage-report</outputDirectory>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>jacoco-merge-unit-and-it</id>
+                        <phase>post-integration-test</phase>
+                        <goals>
+                            <goal>merge</goal>
+                        </goals>
+                        <configuration>
+                            <fileSets>
+                                <fileSet>
+                                    <directory>${project.build.directory}/coverage-reports/</directory>
+                                    <includes>
+                                        <include>*.exec</include>
+                                    </includes>
+                                </fileSet>
+                            </fileSets>
+                            <destFile>${project.build.directory}/coverage-reports/merged.exec</destFile>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>jacoco-report</id>
+                        <phase>post-integration-test</phase>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                        <configuration>
+                            <dataFile>${project.build.directory}/coverage-reports/merged.exec</dataFile>
+                            <outputDirectory>${project.reporting.outputDirectory}/jacoco-merged-test-coverage-report</outputDirectory>
+                        </configuration>
                     </execution>
                 </executions>
             </plugin>
@@ -734,6 +789,9 @@
                         <version>2.3.1</version>
                     </dependency>
                 </dependencies>
+                <configuration>
+                    <jacocoReports>${project.reporting.outputDirectory}/jacoco-merged-test-coverage-report/jacoco.xml</jacocoReports>
+                </configuration>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -743,6 +801,7 @@
                     <!-- testsToExclude come from the profile-->
                     <excludedGroups>${testsToExclude}</excludedGroups>
                     <skip>${skipUnitTests}</skip>
+                    <argLine>${surefire.jacoco.args} ${argLine}</argLine>
                 </configuration>
             </plugin>
             <!-- Run mvn verify to execute these Testcontainers based integration tests. Needs Docker! -->
@@ -751,6 +810,7 @@
                 <artifactId>maven-failsafe-plugin</artifactId>
                 <configuration>
                     <groups>testcontainers</groups>
+                    <argLine>${failsafe.jacoco.args} ${argLine}</argLine>
                 </configuration>
                 <executions>
                     <execution>

From 5975e268ef1150ef76edd589e1ce35a74e683ccf Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 00:23:54 +0200
Subject: [PATCH 113/922] ci: restructure Maven tests to include integration
 tests

---
 .github/workflows/maven_unit_test.yml | 78 +++++++++++++++++++++++++--
 1 file changed, 75 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 629a45a6c2c..8d51702be6b 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -33,6 +33,7 @@ jobs:
         continue-on-error: ${{ matrix.experimental }}
         runs-on: ubuntu-latest
         steps:
+          # Basic setup chores
           - uses: actions/checkout@v2
           - name: Set up JDK ${{ matrix.jdk }}
             uses: actions/setup-java@v2
@@ -45,13 +46,84 @@ jobs:
                 path: ~/.m2
                 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
                 restore-keys: ${{ runner.os }}-m2
+
           - name: Build with Maven
             run: mvn -DcompilerArgument=-Xlint:unchecked -Dtarget.java.version=${{ matrix.jdk }} -P all-unit-tests clean test
-          - name: Maven Code Coverage
-            env:
+
+          # Store the build for the next step (integration test) to avoid recompilation and to transfer coverage reports
+          - run: tar -cvf java-builddir.tar target
+          - uses: actions/upload-artifact@v3
+            with:
+                name: java-builddir
+                path: java-builddir.tar
+                retention-days: 3
+    integration-test:
+        runs-on: ubuntu-latest
+        needs: unittest
+        steps:
+            # Basic setup chores
+            - uses: actions/checkout@v3
+            - name: Set up JDK ${{ matrix.jdk }}
+              uses: actions/setup-java@v2
+              with:
+                  java-version: ${{ matrix.jdk }}
+                  distribution: 'adopt'
+            - name: Cache Maven packages
+              uses: actions/cache@v2
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Get the build output from the unit test job
+            - uses: actions/download-artifact@v3
+              with:
+                  name: java-builddir
+            - run: tar -xvf java-builddir.tar; ls -laR target
+
+            # Run integration tests
+            - run: mvn -Dtarget.java.version=${{ matrix.jdk }} verify
+
+            # Wrap up and send to coverage job
+            - run: tar -cvf java-builddir.tar target
+            - uses: actions/upload-artifact@v3
+              with:
+                  name: java-builddir
+                  path: java-builddir.tar
+                  retention-days: 3
+    coverage-report:
+        runs-on: ubuntu-latest
+        needs: integration-test
+        steps:
+            # Basic setup chores
+            - uses: actions/checkout@v3
+            - name: Set up JDK ${{ matrix.jdk }}
+              uses: actions/setup-java@v2
+              with:
+                  java-version: ${{ matrix.jdk }}
+                  distribution: 'adopt'
+            - name: Cache Maven packages
+              uses: actions/cache@v2
+              with:
+                  path: ~/.m2
+                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+                  restore-keys: ${{ runner.os }}-m2
+
+            # Get the build output from the integration test job
+            - uses: actions/download-artifact@v3
+              with:
+                  name: java-builddir
+            - run: tar -xvf java-builddir.tar; ls -laR target
+
+            # Deposit Code Coverage
+            - name: Maven Code Coverage
+              env:
                 CI_NAME: github
                 COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
-            run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
+              run: mvn -V -B coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
+
+            # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ...
+
     push-app-img:
         name: Publish App Image
         permissions:

From 3b7aa106d118f7cecaefd7680b7a6dd7df2fc670 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 00:29:47 +0200
Subject: [PATCH 114/922] chore(ci): update actions/setup-java to v3 for Maven
 workflow

---
 .github/workflows/maven_unit_test.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 8d51702be6b..1ff08705e36 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -34,9 +34,9 @@ jobs:
         runs-on: ubuntu-latest
         steps:
           # Basic setup chores
-          - uses: actions/checkout@v2
+          - uses: actions/checkout@v3
           - name: Set up JDK ${{ matrix.jdk }}
-            uses: actions/setup-java@v2
+            uses: actions/setup-java@v3
             with:
                 java-version: ${{ matrix.jdk }}
                 distribution: 'adopt'
@@ -64,7 +64,7 @@ jobs:
             # Basic setup chores
             - uses: actions/checkout@v3
             - name: Set up JDK ${{ matrix.jdk }}
-              uses: actions/setup-java@v2
+              uses: actions/setup-java@v3
               with:
                   java-version: ${{ matrix.jdk }}
                   distribution: 'adopt'
@@ -97,10 +97,9 @@ jobs:
         steps:
             # Basic setup chores
             - uses: actions/checkout@v3
-            - name: Set up JDK ${{ matrix.jdk }}
-              uses: actions/setup-java@v2
+            - uses: actions/setup-java@v3
               with:
-                  java-version: ${{ matrix.jdk }}
+                  java-version: '11'
                   distribution: 'adopt'
             - name: Cache Maven packages
               uses: actions/cache@v2

From 2c0a6aabf5852a441009aca39625e8a71eed66b9 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 00:30:13 +0200
Subject: [PATCH 115/922] ci(test): fix missing build matrix for integration
 test job

---
 .github/workflows/maven_unit_test.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 1ff08705e36..3c1c5e7e3e2 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -60,6 +60,23 @@ jobs:
     integration-test:
         runs-on: ubuntu-latest
         needs: unittest
+        name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Integration Tests
+        strategy:
+            fail-fast: false
+            matrix:
+                jdk: [ '11' ]
+                experimental: [ false ]
+                status: [ "Stable" ]
+                #
+                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
+                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
+                # (This is a limitation of how Github is currently handling these things.)
+                #
+                #include:
+                #    - jdk: '17'
+                #      experimental: true
+                #      status: "Experimental"
+        continue-on-error: ${{ matrix.experimental }}
         steps:
             # Basic setup chores
             - uses: actions/checkout@v3

From 8a6f23166870c9460c635535cb021029d45509f5 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 00:43:36 +0200
Subject: [PATCH 116/922] ci(test): tweak artifact upload and job names

---
 .github/workflows/maven_unit_test.yml | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 3c1c5e7e3e2..a4753f24668 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -1,4 +1,4 @@
-name: Maven Unit Tests
+name: Maven Tests
 
 on:
     push:
@@ -96,21 +96,22 @@ jobs:
             - uses: actions/download-artifact@v3
               with:
                   name: java-builddir
-            - run: tar -xvf java-builddir.tar; ls -laR target
+            - run: tar -xvf java-builddir.tar
 
-            # Run integration tests
-            - run: mvn -Dtarget.java.version=${{ matrix.jdk }} verify
+            # Run integration tests (but not unit tests again)
+            - run: mvn -DskipUnitTests -Dtarget.java.version=${{ matrix.jdk }} verify
 
             # Wrap up and send to coverage job
-            - run: tar -cvf java-builddir.tar target
+            - run: tar -cvf java-reportdir.tar target/site
             - uses: actions/upload-artifact@v3
               with:
-                  name: java-builddir
-                  path: java-builddir.tar
+                  name: java-reportdir
+                  path: java-reportdir.tar
                   retention-days: 3
     coverage-report:
         runs-on: ubuntu-latest
         needs: integration-test
+        name: Coverage Report Submission
         steps:
             # Basic setup chores
             - uses: actions/checkout@v3
@@ -128,8 +129,8 @@ jobs:
             # Get the build output from the integration test job
             - uses: actions/download-artifact@v3
               with:
-                  name: java-builddir
-            - run: tar -xvf java-builddir.tar; ls -laR target
+                  name: java-reportdir
+            - run: tar -xvf java-reportdir.tar
 
             # Deposit Code Coverage
             - name: Maven Code Coverage

From edef7d5bd52f3625b2e15521046d97a73f999894 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 01:20:04 +0200
Subject: [PATCH 117/922] docs(config): add OIDC to JVM options list with
 backlink

---
 doc/sphinx-guides/source/installation/config.rst | 7 +++++++
 doc/sphinx-guides/source/installation/oidc.rst   | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f8aef8c59da..4b1a8bd14b3 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -2420,6 +2420,13 @@ Can also be set via any `supported MicroProfile Config API source`_, e.g. the en
 ``DATAVERSE_UI_SHOW_VALIDITY_FILTER``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions.
 
 
+dataverse.auth.oidc.*
++++++++++++++++++++++
+
+Provision a single :doc:`OpenID Connect authentication provider <oidc>` using MicroProfile Config. You can find a list of
+all available options at :ref:`oidc-mpconfig`.
+
+
 .. _feature-flags:
 
 Feature Flags
diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index 9848d73b189..0dc5ca4ff4a 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -116,6 +116,8 @@ The Dataverse installation will automatically try to load the provider and retri
 You should see the new provider under "Other options" on the Log In page, as described in the :doc:`/user/account`
 section of the User Guide.
 
+.. _oidc-mpconfig:
+
 Provision via MPCONFIG
 ^^^^^^^^^^^^^^^^^^^^^^
 

From e91a046a790506e959ffcc9f71d27f6207b4d4e1 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 01:51:04 +0200
Subject: [PATCH 118/922] feat(oidc): replace map-based cache of PKCE verifiers

Instead of using an unlimited growth Map of verifiers
as a cache, we now will use a real evicting cache implementation
to limit the size and age of entries.

This will limit resource waste and mitigate an unlikely
but present attack vector of pumping up the cache to DDoS us.
---
 pom.xml                                            |  7 +++++++
 .../providers/oauth2/oidc/OIDCAuthProvider.java    | 14 ++++++++++++--
 .../harvard/iq/dataverse/settings/JvmSettings.java |  2 ++
 .../META-INF/microprofile-config.properties        |  4 ++++
 4 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5d1523e01b8..0484a80ddd9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -386,6 +386,13 @@
             <artifactId>oauth2-oidc-sdk</artifactId>
             <version>10.9.1</version>
         </dependency>
+        <!-- Caching library, current main use case is for OIDC authentication -->
+        <dependency>
+            <groupId>com.github.ben-manes.caffeine</groupId>
+            <artifactId>caffeine</artifactId>
+            <version>3.1.6</version>
+        </dependency>
+        
         <!-- New and Improved GDCC XOAI library! --> 
         <dependency>
             <groupId>io.gdcc</groupId>
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
index 52362f7abeb..818332ea282 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
@@ -1,5 +1,7 @@
 package edu.harvard.iq.dataverse.authorization.providers.oauth2.oidc;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.github.scribejava.core.builder.api.DefaultApi20;
 import com.nimbusds.oauth2.sdk.AuthorizationCode;
 import com.nimbusds.oauth2.sdk.AuthorizationCodeGrant;
@@ -36,10 +38,13 @@
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2Exception;
 import edu.harvard.iq.dataverse.authorization.providers.oauth2.OAuth2UserRecord;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 
 import java.io.IOException;
 import java.net.URI;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -69,8 +74,13 @@ public class OIDCAuthProvider extends AbstractOAuth2AuthenticationProvider {
     /**
      * Using PKCE, we create and send a special {@link CodeVerifier}. This contains a secret
      * we need again when verifying the response by the provider, thus the cache.
+     * To be absolutely sure this may not be abused to DDoS us and not let unused verifiers rot,
+     * use an evicting cache implementation and not a standard map.
      */
-    private final Map<String,CodeVerifier> verifierCache = new ConcurrentHashMap<>();
+    private final Cache<String,CodeVerifier> verifierCache = Caffeine.newBuilder()
+        .maximumSize(JvmSettings.OIDC_PKCE_CACHE_MAXSIZE.lookup(Integer.class))
+        .expireAfterWrite(Duration.of(JvmSettings.OIDC_PKCE_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.SECONDS))
+        .build();
     
     public OIDCAuthProvider(String aClientId, String aClientSecret, String issuerEndpointURL,
                             boolean pkceEnabled, String pkceMethod) throws AuthorizationSetupException {
@@ -201,7 +211,7 @@ public String buildAuthzUrl(String state, String callbackUrl) {
     public OAuth2UserRecord getUserRecord(String code, String state, String redirectUrl) throws IOException, OAuth2Exception {
         // Retrieve the verifier from the cache and clear from the cache. If not found, will be null.
         // Will be sent to token endpoint for verification, so if required but missing, will lead to exception.
-        CodeVerifier verifier = verifierCache.remove(state);
+        CodeVerifier verifier = verifierCache.getIfPresent(state);
         
         // Create grant object - again, this is null-safe for the verifier
         AuthorizationGrant codeGrant = new AuthorizationCodeGrant(
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index 144be72c91a..1122b64c139 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -125,6 +125,8 @@ public enum JvmSettings {
     SCOPE_OIDC_PKCE(SCOPE_OIDC, "pkce"),
     OIDC_PKCE_ENABLED(SCOPE_OIDC_PKCE, "enabled"),
     OIDC_PKCE_METHOD(SCOPE_OIDC_PKCE, "method"),
+    OIDC_PKCE_CACHE_MAXSIZE(SCOPE_OIDC_PKCE, "max-cache-size"),
+    OIDC_PKCE_CACHE_MAXAGE(SCOPE_OIDC_PKCE, "max-cache-age"),
 
     // UI SETTINGS
     SCOPE_UI(PREFIX, "ui"),
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 3e166d0527f..38a4d8df0ab 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -54,3 +54,7 @@ dataverse.pid.datacite.rest-api-url=https://api.test.datacite.org
 
 # Handle.Net
 dataverse.pid.handlenet.index=300
+
+# AUTHENTICATION
+dataverse.auth.oidc.pkce.max-cache-size=10000
+dataverse.auth.oidc.pkce.max-cache-age=300
\ No newline at end of file

From 8d4a75e8236298d787fdf738512c615c13c3654e Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 01:51:39 +0200
Subject: [PATCH 119/922] docs(oidc): describe new config options for PKCE
 verifier cache

---
 doc/sphinx-guides/source/installation/oidc.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index 0dc5ca4ff4a..e036e9c8470 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -168,3 +168,12 @@ The following options are available:
     - A subtitle, currently not displayed by the UI.
     - N
     - ``OpenID Connect``
+  * - ``dataverse.auth.oidc.pkce.max-cache-size``
+    - Tune the maximum size of all OIDC providers' verifier cache (= number of outstanding PKCE-enabled auth responses).
+    - N
+    - 10000
+  * - ``dataverse.auth.oidc.pkce.max-cache-age``
+    - Tune the maximum age of all OIDC providers' verifier cache entries. Default is 5 minutes, equivalent to lifetime
+      of many OIDC access tokens.
+    - N
+    - 300
\ No newline at end of file

From 8ca25fff4f4e1801c97ff28ba1947e4d05bfc915 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 24 May 2023 01:56:47 +0200
Subject: [PATCH 120/922] docs(oidc,test): add release note

---
 .../9268-8349-oidc-improvements.md            | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 doc/release-notes/9268-8349-oidc-improvements.md

diff --git a/doc/release-notes/9268-8349-oidc-improvements.md b/doc/release-notes/9268-8349-oidc-improvements.md
new file mode 100644
index 00000000000..cb0a9685c69
--- /dev/null
+++ b/doc/release-notes/9268-8349-oidc-improvements.md
@@ -0,0 +1,28 @@
+## OpenID Connect Authentication Provider Improvements
+
+### Using MicroProfile Config For Provisioning
+
+With this release it is possible to provision a single OIDC-based authentication provider
+by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
+
+If you are using an external OIDC provider component as identity management system and/or broker
+to other authentication providers such as Google, eduGain SAML and so on, this might make your
+life easier during instance setups and reconfiguration. You no longer need to generate the
+necessary JSON file.
+
+### Adding PKCE Support
+
+Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
+support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
+
+## Improved Testing
+
+With this release, we add a new type of testing to Dataverse: integration tests which are no end-to-end tests
+like our API tests. Starting with OIDC authentication support, we test regularly on CI for working condition
+of both OIDC login options in UI and API.
+
+The testing and development Keycloak realm has been updated with more users and compatibility with Keycloak 21.
+
+The support for setting JVM options during testing has been improved for developers. You now may add the
+`@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
+also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.

From 8b2937e80e53528106d12ea7e8ed7204c7b7ee2c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 25 May 2023 17:35:23 -0400
Subject: [PATCH 121/922] Add call to populate file lists

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index b8d2507bc6b..c857d00a27d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -370,6 +370,7 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
 
     public void setTermsGuestbookPopupAction(String popupAction){
         if(popupAction != null && popupAction.length() > 0){
+            logger.info("TGPA set to " + popupAction);
             this.termsGuestbookPopupAction = popupAction;
         }
 
@@ -5178,6 +5179,9 @@ public boolean isFileAccessRequestMultiButtonEnabled(){
         if (!isSessionUserAuthenticated() || !dataset.isFileAccessRequest()){
             return false;
         }
+        //populate file lists
+        filterSelectedFiles();
+        
         if( this.selectedRestrictedFiles == null || this.selectedRestrictedFiles.isEmpty() ){
             return false;
         }

From 7ddc0466a767b292e8dfe6625feea770b112ded5 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 25 May 2023 17:35:42 -0400
Subject: [PATCH 122/922] hide accept terms buttons in download case

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 5616fa48280..bdaa6d92432 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -41,7 +41,7 @@
                 </div>
             </div>
         </div>
-        <div class="button-block">
+        <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle.download}">
             <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" 
                            action="#{fileDownloadHelper.requestAccessIndirect()}"    
                            update="@([id$=requestAccessConsolidated]), @([id$=requestPanel]), @([id$=messagePanel]) @([id$=filesTable])" oncomplete="PF('requestAccessPopup').hide();">
@@ -51,6 +51,7 @@
             </button>
         </div>
         <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle['file.requestAccess']}">
+        <!--TODO - add "Accept and" if ToA exist?-->
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"                          
                     update="@([id$=Messages])"

From 2893b62852f11ad0496b88562394f10efc3f2733 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 25 May 2023 17:49:17 -0400
Subject: [PATCH 123/922] handle changes from #6919

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index bdaa6d92432..9c3391ef9ae 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -60,7 +60,7 @@
             </p:commandLink>
 
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{!workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
-                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(fileDownloadHelper.guestbookResponse)}"                          
+                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"
                     update="@([id$=Messages])"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
                     <f:param name="DO_GB_VALIDATION" value="true"/>

From 90186edc47772a2aa39089dc1607caecf3c3917a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 25 May 2023 19:47:11 -0400
Subject: [PATCH 124/922] fix query

---
 .../edu/harvard/iq/dataverse/DataFileServiceBean.java    | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 449e8d351c6..f4a33e4f12f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -861,10 +861,11 @@ private List<AuthenticatedUser> retrieveFileAccessRequesters(DataFile fileIn) {
         // List<Object> requesters = em.createNativeQuery("select authenticated_user_id
         // from fileaccessrequests where datafile_id =
         // "+fileIn.getId()).getResultList();
-        List<Object> requesters = em.createNativeQuery("select authenticated_user_id from fileaccessrequests where datafile_id = " + fileIn.getId() + " and request_state='CREATED'").getResultList();
-
-        for (Object userIdObj : requesters) {
-            Long userId = (Long) userIdObj;
+        TypedQuery<Long> typedQuery = em.createQuery("select f.user.id from FileAccessRequest f where f.dataFile.id = :file_id and f.requestState= :requestState", Long.class);
+        typedQuery.setParameter("file_id", fileIn.getId());
+        typedQuery.setParameter("requestState", FileAccessRequest.RequestState.CREATED);
+        List<Long> requesters = typedQuery.getResultList();
+        for (Long userId : requesters) {
             AuthenticatedUser user = userService.find(userId);
             if (user != null) {
                 retList.add(user);

From 728c886444161cd23ad060ab8665961f4452ace6 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Jun 2023 01:16:08 +0200
Subject: [PATCH 125/922] ci: add installed gdcc packages to Maven run artifact

---
 .github/workflows/maven_unit_test.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 51e3c5188f6..cc918e16d97 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -60,11 +60,15 @@ jobs:
           - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
 
           # Store the build for the next step (integration test) to avoid recompilation and to transfer coverage reports
-          - run: tar -cvf java-builddir.tar target
+          - run: |
+                tar -cvf java-builddir.tar target
+                tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-*
           - uses: actions/upload-artifact@v3
             with:
-                name: java-builddir
-                path: java-builddir.tar
+                name: java-artifacts
+                path: |
+                    java-builddir.tar
+                    java-m2-selection.tar
                 retention-days: 3
 
     integration-test:
@@ -100,11 +104,12 @@ jobs:
             # Get the build output from the unit test job
             - uses: actions/download-artifact@v3
               with:
-                  name: java-builddir
-            - run: tar -xvf java-builddir.tar
+                  name: java-artifacts
+            - run: |
+                tar -xvf java-builddir.tar
+                tar -xvf java-m2-selection.tar -C /
 
             # Run integration tests (but not unit tests again)
-            # TODO - adopt to parent module
             - run: mvn -DskipUnitTests -Dtarget.java.version=${{ matrix.jdk }} verify
 
             # Wrap up and send to coverage job
@@ -148,6 +153,7 @@ jobs:
                 jacoco:report coveralls:report
 
           # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ...
+
     push-app-img:
         name: Publish App Image
         permissions:

From 77242777e2afa68335884d9eed2f68e1be750102 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Jun 2023 08:39:03 +0200
Subject: [PATCH 126/922] ci: some more fixes for Maven Tests

---
 .github/workflows/maven_unit_test.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index cc918e16d97..5a5e55f82aa 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -47,12 +47,14 @@ jobs:
                 cache: maven
 
           # The reason why we use "install" here is that we want the submodules to be available in the next step.
-          # Also, we can cache them this way for jobs triggered by this one.
-          - name: Build with Maven
+          # Also, we can cache them this way for jobs triggered by this one. We need to skip ITs here, as we run
+          # them in the next job - but install usually runs through verify phase.
+          - name: Build with Maven and run unit tests
             run: > 
                 mvn -B -f modules/dataverse-parent
                 -Dtarget.java.version=${{ matrix.jdk }}
                 -DcompilerArgument=-Xlint:unchecked -P all-unit-tests
+                -DskipIntegrationTests
                 -pl edu.harvard.iq:dataverse -am
                 install
 
@@ -127,10 +129,9 @@ jobs:
         steps:
             # Basic setup chores
           - uses: actions/checkout@v3
-          - name: Set up JDK ${{ matrix.jdk }}
-            uses: actions/setup-java@v3
+          - uses: actions/setup-java@v3
             with:
-                java-version: ${{ matrix.jdk }}
+                java-version: '11'
                 distribution: temurin
                 cache: maven
 
@@ -141,7 +142,7 @@ jobs:
           - run: tar -xvf java-reportdir.tar
 
           # Deposit Code Coverage
-          - name: Maven Code Coverage
+          - name: Deposit Code Coverage
             env:
                 CI_NAME: github
                 COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}

From 0287e037e6b76be83006c8447a7007d2386aea91 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Jun 2023 08:46:48 +0200
Subject: [PATCH 127/922] build: add config to enable skipping jacoco and
 failsafe tests

---
 pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pom.xml b/pom.xml
index fd92ccdc06e..2aa74657422 100644
--- a/pom.xml
+++ b/pom.xml
@@ -747,6 +747,7 @@
                         <configuration>
                             <destFile>${project.build.directory}/coverage-reports/jacoco-integration.exec</destFile>
                             <propertyName>failsafe.jacoco.args</propertyName>
+                            <skip>${skipIntegrationTests}</skip>
                         </configuration>
                     </execution>
                     <execution>
@@ -758,6 +759,7 @@
                         <configuration>
                             <dataFile>${project.build.directory}/coverage-reports/jacoco-integration.exec</dataFile>
                             <outputDirectory>${project.reporting.outputDirectory}/jacoco-integration-test-coverage-report</outputDirectory>
+                            <skip>${skipIntegrationTests}</skip>
                         </configuration>
                     </execution>
                     <execution>
@@ -776,6 +778,7 @@
                                 </fileSet>
                             </fileSets>
                             <destFile>${project.build.directory}/coverage-reports/merged.exec</destFile>
+                            <skip>${skipIntegrationTests}</skip>
                         </configuration>
                     </execution>
                     <execution>
@@ -787,6 +790,7 @@
                         <configuration>
                             <dataFile>${project.build.directory}/coverage-reports/merged.exec</dataFile>
                             <outputDirectory>${project.reporting.outputDirectory}/jacoco-merged-test-coverage-report</outputDirectory>
+                            <skip>${skipIntegrationTests}</skip>
                         </configuration>
                     </execution>
                 </executions>
@@ -824,6 +828,7 @@
                 <configuration>
                     <groups>testcontainers</groups>
                     <argLine>${failsafe.jacoco.args} ${argLine}</argLine>
+                    <skip>${skipIntegrationTests}</skip>
                 </configuration>
                 <executions>
                     <execution>

From 28ddc3797b60eafb2b4144bb06610dc1caadf54f Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Jun 2023 11:08:25 +0200
Subject: [PATCH 128/922] fix(auth,oidc): do not add null verifier to cache

Will throw an NPE otherwise.
---
 .../authorization/providers/oauth2/oidc/OIDCAuthProvider.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
index 818332ea282..5eb2b391eb7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthProvider.java
@@ -191,7 +191,9 @@ public String buildAuthzUrl(String state, String callbackUrl) {
         
         // Cache the PKCE verifier, as we need the secret in it for verification later again, after the client sends us
         // the auth code! We use the state to cache the verifier, as the state is unique per authentication event.
-        this.verifierCache.put(state, pkceVerifier);
+        if (pkceVerifier != null) {
+            this.verifierCache.put(state, pkceVerifier);
+        }
         
         return req.toURI().toString();
     }

From 749c13be13c6c6e3c0f8bea26674e488cd62d6ab Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Jun 2023 11:10:09 +0200
Subject: [PATCH 129/922] feat(ct,oidc,auth): add dev keycloak connection to
 compose file

---
 docker-compose-dev.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index e3f93b77d4a..d582a6375f9 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -13,6 +13,10 @@ services:
       - DATAVERSE_DB_PASSWORD=secret
       - DATAVERSE_DB_USER=${DATAVERSE_DB_USER}
       - DATAVERSE_FEATURE_API_BEARER_AUTH=1
+      - DATAVERSE_AUTH_OIDC_ENABLED=1
+      - DATAVERSE_AUTH_OIDC_CLIENT_ID=test
+      - DATAVERSE_AUTH_OIDC_CLIENT_SECRET=94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8
+      - DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL=http://keycloak.mydomain.com:8090/realms/test
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4848:4848" # HTTP (Payara Admin Console)

From 849df5d2214a630fe70dc177c7188f09b4ae782b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 6 Jun 2023 13:04:30 +0200
Subject: [PATCH 130/922] docs,fix(oidc): fix API auth docs example with new
 test realm

---
 doc/sphinx-guides/source/api/auth.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst
index bbc81b595e3..eced7afbbcf 100644
--- a/doc/sphinx-guides/source/api/auth.rst
+++ b/doc/sphinx-guides/source/api/auth.rst
@@ -77,6 +77,6 @@ To test if bearer tokens are working, you can try something like the following (
 
 .. code-block:: bash
 
-  export TOKEN=`curl -s -X POST --location "http://keycloak.mydomain.com:8090/realms/oidc-realm/protocol/openid-connect/token" -H "Content-Type: application/x-www-form-urlencoded" -d "username=kcuser&password=kcpassword&grant_type=password&client_id=oidc-client&client_secret=ss6gE8mODCDfqesQaSG3gwUwZqZt547E" | jq '.access_token' -r | tr -d "\n"`
+  export TOKEN=`curl -s -X POST --location "http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token" -H "Content-Type: application/x-www-form-urlencoded" -d "username=user&password=user&grant_type=password&client_id=test&client_secret=94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8" | jq '.access_token' -r | tr -d "\n"`
   
   curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/api/users/:me

From 502e660fe342939a617edd6d17a425c83b5a269b Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 12 May 2023 13:22:46 -0400
Subject: [PATCH 131/922] suppress thumb generation after a failure

---
 .../edu/harvard/iq/dataverse/DvObject.java    | 14 +++++
 .../dataaccess/ImageThumbConverter.java       | 55 ++++++++++++-------
 .../dataverse/ingest/IngestServiceBean.java   |  4 +-
 .../V5.13.0.1__9506-track-thumb-failures.sql  |  1 +
 4 files changed, 54 insertions(+), 20 deletions(-)
 create mode 100644 src/main/resources/db/migration/V5.13.0.1__9506-track-thumb-failures.sql

diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
index 854888737ee..6cb3816e3f1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
@@ -181,7 +181,20 @@ public boolean isPreviewImageAvailable() {
     public void setPreviewImageAvailable(boolean status) {
         this.previewImageAvailable = status;
     }
+    
+    /** Indicates whether a previous attempt to generate a preview image has failed, regardless of size.
+     * If so, we won't want to try again every time the preview/thumbnail is requested for a view.
+     */
+    private boolean previewsHaveFailed;
+
+    public boolean isPreviewsHaveFailed() {
+        return previewsHaveFailed;
+    }
 
+    public void setPreviewsHaveFailed(boolean previewsHaveFailed) {
+        this.previewsHaveFailed = previewsHaveFailed;
+    }
+    
     public Timestamp getModificationTime() {
         return modificationTime;
     }
@@ -462,6 +475,7 @@ public void setStorageIdentifier(String storageIdentifier) {
      */
     public abstract boolean isAncestorOf( DvObject other );
     
+
     @OneToMany(mappedBy = "definitionPoint",cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST}, orphanRemoval=true)
     List<RoleAssignment> roleAssignments;
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index 2b4aed3a9a5..eb08646454d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -48,6 +48,7 @@
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.commons.io.IOUtils;
 //import org.primefaces.util.Base64;
@@ -110,15 +111,24 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
         }
 
         if (isThumbnailCached(storageIO, size)) {
+            logger.fine("Found cached thumbnail for " + file.getId());
             return true;
         }
 
-        logger.fine("Checking for thumbnail, file type: " + file.getContentType());
-
-        if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
-            return generateImageThumbnail(storageIO, size);
-        } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
-            return generatePDFThumbnail(storageIO, size);
+        logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + "to generate thumbnail, file id: " + file.getId());
+        // Don't try to generate if there have been failures:
+        if (!file.isPreviewsHaveFailed()) {
+            boolean thumbnailGenerated = false;
+            if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
+                thumbnailGenerated = generateImageThumbnail(storageIO, size);
+            } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
+                thumbnailGenerated = generatePDFThumbnail(storageIO, size);
+            }
+            if (!thumbnailGenerated) {
+                logger.fine("No thumbnail generated for " + file.getId());
+                file.setPreviewGenerationHasPreviouslyFailed(true);
+            }
+            return thumbnailGenerated;
         }
 
         return false;
@@ -436,20 +446,27 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) {
         if (cachedThumbnailChannel == null) {
             logger.fine("Null channel for aux object " + THUMBNAIL_SUFFIX + size);
 
-            // try to generate, if not available: 
-            boolean generated = false;
-            if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
-                generated = generateImageThumbnail(storageIO, size);
-            } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
-                generated = generatePDFThumbnail(storageIO, size);
-            }
+            // try to generate, if not available and hasn't failed before
+            logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + "to generate base64 thumbnail, file id: " + file.getId());
+            if (!file.isPreviewsHaveFailed()) {
+                boolean generated = false;
+                if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
+                    generated = generateImageThumbnail(storageIO, size);
+                } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
+                    generated = generatePDFThumbnail(storageIO, size);
+                }
 
-            if (generated) {
-                // try to open again: 
-                try {
-                    cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size);
-                } catch (Exception ioEx) {
-                    cachedThumbnailChannel = null;
+                if (!generated) {
+                    // Record failure
+                    logger.fine("Failed to generate base64 thumbnail for file id: " + file.getId());
+                    file.setPreviewGenerationHasPreviouslyFailed(true);
+                } else {
+                    // Success - try to open again:
+                    try {
+                        cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size);
+                    } catch (Exception ioEx) {
+                        cachedThumbnailChannel = null;
+                    }
                 }
             }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 5a353453fe8..fbe2d7b38ff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -292,7 +292,9 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 
 								} catch (IOException ioex) {
 									logger.warning("Failed to save generated file " + generated.toString());
-								}
+									//Shouldn't mark this file as having a preview after this.
+									dataFile.setPreviewImageAvailable(false);
+                                }
 							}
 
 							// ... but we definitely want to delete it:
diff --git a/src/main/resources/db/migration/V5.13.0.1__9506-track-thumb-failures.sql b/src/main/resources/db/migration/V5.13.0.1__9506-track-thumb-failures.sql
new file mode 100644
index 00000000000..9b12d27db91
--- /dev/null
+++ b/src/main/resources/db/migration/V5.13.0.1__9506-track-thumb-failures.sql
@@ -0,0 +1 @@
+ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS previewshavefailed BOOLEAN DEFAULT FALSE;
\ No newline at end of file

From 0fea5ccca11b2348429ddfee75e4bafc709c7473 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 12 May 2023 13:25:38 -0400
Subject: [PATCH 132/922] refactor error

---
 .../harvard/iq/dataverse/dataaccess/ImageThumbConverter.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index eb08646454d..254c334d655 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -126,7 +126,7 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
             }
             if (!thumbnailGenerated) {
                 logger.fine("No thumbnail generated for " + file.getId());
-                file.setPreviewGenerationHasPreviouslyFailed(true);
+                file.setPreviewsHaveFailed(true);
             }
             return thumbnailGenerated;
         }
@@ -459,7 +459,7 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) {
                 if (!generated) {
                     // Record failure
                     logger.fine("Failed to generate base64 thumbnail for file id: " + file.getId());
-                    file.setPreviewGenerationHasPreviouslyFailed(true);
+                    file.setPreviewsHaveFailed(true);
                 } else {
                     // Success - try to open again:
                     try {

From 8f5350ae0df4df60c55ff770259531935cb6ac9b Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 15 May 2023 10:32:21 -0400
Subject: [PATCH 133/922] cache isThumb available

---
 .../iq/dataverse/ThumbnailServiceWrapper.java       | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 6c8db8c124b..e2bb21c8a4c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -49,6 +49,7 @@ public class ThumbnailServiceWrapper implements java.io.Serializable  {
     
     private Map<Long, String> dvobjectThumbnailsMap = new HashMap<>();
     private Map<Long, DvObject> dvobjectViewMap = new HashMap<>();
+    private Map<Long, Boolean> hasThumbMap = new HashMap<>();
 
     private String getAssignedDatasetImage(Dataset dataset, int size) {
         if (dataset == null) {
@@ -133,7 +134,7 @@ public String getFileCardImageAsBase64Url(SolrSearchResult result) {
 
             if ((!((DataFile)result.getEntity()).isRestricted()
                         || permissionsWrapper.hasDownloadFilePermission(result.getEntity()))
-                    && dataFileService.isThumbnailAvailable((DataFile) result.getEntity())) {
+                    && isThumbnailAvailable((DataFile) result.getEntity())) {
                 
                 cardImageUrl = ImageThumbConverter.getImageThumbnailAsBase64(
                         (DataFile) result.getEntity(),
@@ -159,6 +160,13 @@ public String getFileCardImageAsBase64Url(SolrSearchResult result) {
         return null;
     }
 
+    public boolean isThumbnailAvailable(DataFile entity) {
+        if(!hasThumbMap.containsKey(entity.getId())) {
+            hasThumbMap.put(entity.getId(), dataFileService.isThumbnailAvailable(entity));
+        }
+        return hasThumbMap.get(entity.getId());
+    }
+
     // it's the responsibility of the user - to make sure the search result
     // passed to this method is of the Dataset type!
     public String getDatasetCardImageAsBase64Url(SolrSearchResult result) {
@@ -295,7 +303,7 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
                 }
             }
 
-            if (dataFileService.isThumbnailAvailable(thumbnailImageFile)) {
+            if (isThumbnailAvailable(thumbnailImageFile)) {
                 cardImageUrl = ImageThumbConverter.getImageThumbnailAsBase64(
                         thumbnailImageFile,
                         size);
@@ -323,6 +331,7 @@ public String getDataverseCardImageAsBase64Url(SolrSearchResult result) {
     public void resetObjectMaps() {
         dvobjectThumbnailsMap = new HashMap<>();
         dvobjectViewMap = new HashMap<>();
+        hasThumbMap = new HashMap<>();
     }
 
     

From 8604eef7f470eade8dbf885ed42bc47407db74ff Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 15 May 2023 13:22:18 -0400
Subject: [PATCH 134/922] set thumb fail column

---
 .../java/edu/harvard/iq/dataverse/DataFileServiceBean.java   | 5 ++++-
 .../harvard/iq/dataverse/dataaccess/ImageThumbConverter.java | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 196f84b6877..a5822828682 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -1127,7 +1127,7 @@ public boolean isThumbnailAvailable (DataFile file) {
         }
         
         // If thumbnails are not even supported for this class of files, 
-        // there's notthing to talk about:      
+        // there's nothing to talk about:      
         if (!FileUtil.isThumbnailSupported(file)) {
             return false;
         }
@@ -1149,6 +1149,9 @@ public boolean isThumbnailAvailable (DataFile file) {
            file.setPreviewImageAvailable(true);
            this.save(file); 
            return true;
+       } else {
+           file.setPreviewsHaveFailed(true);
+           this.save(file);
        }
 
        return false;
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index 254c334d655..ab9294eea72 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -115,7 +115,7 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
             return true;
         }
 
-        logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + "to generate thumbnail, file id: " + file.getId());
+        logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId());
         // Don't try to generate if there have been failures:
         if (!file.isPreviewsHaveFailed()) {
             boolean thumbnailGenerated = false;

From aeae8f4ddbb05794c177e9b1d33725e1ed7d7e2f Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 15 May 2023 13:50:49 -0400
Subject: [PATCH 135/922] use thumb wrapper in edit and view files

---
 src/main/webapp/editFilesFragment.xhtml  | 4 ++--
 src/main/webapp/file-info-fragment.xhtml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml
index a4e635b8c14..af06b44e3bc 100644
--- a/src/main/webapp/editFilesFragment.xhtml
+++ b/src/main/webapp/editFilesFragment.xhtml
@@ -360,13 +360,13 @@
                                     <div class="pull-left col-file-thumb">
                                         <div class="thumbnail-block text-center">
                                             <!-- Thumbnail Preview -->
-                                            <span class="file-thumbnail-preview-img" jsf:rendered="#{!empty fileMetadata.dataFile.id and dataFileServiceBean.isThumbnailAvailable(fileMetadata.dataFile)}"
+                                            <span class="file-thumbnail-preview-img" jsf:rendered="#{!empty fileMetadata.dataFile.id and thumbnailServiceWrapper.isThumbnailAvailable(fileMetadata.dataFile)}"
                                                   data-container="body" data-toggle="popover" data-placement="top" data-trigger="hover" data-html="true" data-content="&lt;img src=&#34;/api/access/datafile/#{fileMetadata.dataFile.id}?imageThumb=400&#34; alt=&#34;#{bundle['file.preview']} #{fileMetadata.label}&#34; /&gt;"
                                                   data-template='&lt;div class="popover thumb-preview" role="tooltip"&gt;&lt;div class="arrow"&gt;&lt;/div&gt;&lt;h3 class="popover-title"&gt;&lt;/h3&gt;&lt;div class="popover-content"&gt;&lt;/div&gt;&lt;/div&gt;'>
                                                 <p:graphicImage value="/api/access/datafile/#{fileMetadata.dataFile.id}?imageThumb=true" alt="#{fileMetadata.label}"/>
                                             </span>
                                             <!-- Default Icon -->
-                                            <span class="icon-#{dataFileServiceBean.getFileThumbnailClass(fileMetadata.dataFile)} file-thumbnail-icon text-muted" jsf:rendered="#{(!empty fileMetadata.dataFile.id and !dataFileServiceBean.isThumbnailAvailable(fileMetadata.dataFile)) or (empty fileMetadata.dataFile.id and !fileMetadata.dataFile.previewImageAvailable)}"/>
+                                            <span class="icon-#{dataFileServiceBean.getFileThumbnailClass(fileMetadata.dataFile)} file-thumbnail-icon text-muted" jsf:rendered="#{(!empty fileMetadata.dataFile.id and !thumbnailServiceWrapper.isThumbnailAvailable(fileMetadata.dataFile)) or (empty fileMetadata.dataFile.id and !fileMetadata.dataFile.previewImageAvailable)}"/>
                                             <ui:fragment rendered="#{empty fileMetadata.dataFile.id and !empty fileMetadata.dataFile.storageIdentifier and fileMetadata.dataFile.previewImageAvailable}">
                                                 <img src="#{EditDatafilesPage.getTemporaryPreviewAsBase64(fileMetadata.dataFile.storageIdentifier)}" alt="#{fileMetadata.label}"/>
                                                 <h:outputText id="imgPreview" value="#{bundle['preview']}" styleClass="bg-info text-info text-center show"/>
diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml
index 33a8d2c3ca5..3e8e80d51e7 100644
--- a/src/main/webapp/file-info-fragment.xhtml
+++ b/src/main/webapp/file-info-fragment.xhtml
@@ -28,8 +28,8 @@
     <p:outputPanel id="fileInfoInclude-filesTable" styleClass="media" rendered="#{fileMetadata != null}">
         <div class="media-left col-file-thumb" style="padding-top:4px;">
             <div class="media-object thumbnail-block text-center">
-                <span class="icon-#{dataFileServiceBean.getFileThumbnailClass(fileMetadata.dataFile)} file-thumbnail-icon text-muted" jsf:rendered="#{!fileDownloadHelper.canDownloadFile(fileMetadata) or !dataFileServiceBean.isThumbnailAvailable(fileMetadata.dataFile)}"/>
-                <span class="file-thumbnail-preview-img" jsf:rendered="#{fileDownloadHelper.canDownloadFile(fileMetadata) and dataFileServiceBean.isThumbnailAvailable(fileMetadata.dataFile)}"
+                <span class="icon-#{dataFileServiceBean.getFileThumbnailClass(fileMetadata.dataFile)} file-thumbnail-icon text-muted" jsf:rendered="#{!fileDownloadHelper.canDownloadFile(fileMetadata) or !thumbnailServiceWrapper.isThumbnailAvailable(fileMetadata.dataFile)}"/>
+                <span class="file-thumbnail-preview-img" jsf:rendered="#{fileDownloadHelper.canDownloadFile(fileMetadata) and thumbnailServiceWrapper.isThumbnailAvailable(fileMetadata.dataFile)}"
                       data-toggle="popover" data-placement="auto top" data-trigger="hover" data-html="true" data-content="&lt;img src=&#34;/api/access/datafile/#{fileMetadata.dataFile.id}?imageThumb=400&#34; alt=&#34;#{bundle['file.preview']} #{fileMetadata.label}&#34; /&gt;"
                       data-template='&lt;div class="popover thumb-preview" role="tooltip"&gt;&lt;div class="arrow"&gt;&lt;/div&gt;&lt;h3 class="popover-title"&gt;&lt;/h3&gt;&lt;div class="popover-content"&gt;&lt;/div&gt;&lt;/div&gt;'>
                     <p:graphicImage value="/api/access/datafile/#{fileMetadata.dataFile.id}?imageThumb=true" alt="#{fileMetadata.label}"/>

From c4ad20bc4b67b93908e60b76a251240f4a6e2540 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 17 May 2023 13:49:35 -0400
Subject: [PATCH 136/922] add api

---
 .../edu/harvard/iq/dataverse/api/Admin.java   | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index d219339add9..14c556e9caa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -2321,4 +2321,26 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
         return ok(Json.createObjectBuilder().add(ExternalToolHandler.SIGNED_URL, signedUrl));
     }
  
+    @DELETE
+    @Path("/clearThumbnailFailureFlag")
+    public Response clearThumbnailFailureFlag() {
+        em.createNativeQuery("UPDATE dvobject SET previewshavefailed = FALSE").executeUpdate();
+        return ok("Thumnail Failure Flags cleared.");
+    }
+    
+    @DELETE
+    @Path("/clearThumbnailFailureFlag/{id}")
+    public Response clearThumbnailFailureFlagByDatafile(@PathParam("id") String fileId) {
+        try {
+            DataFile df = findDataFileOrDie(fileId);
+            Query deleteQuery = em.createNativeQuery("UPDATE dvobject SET previewshavefailed = FALSE where id = ?");
+            deleteQuery.setParameter(1, df.getId());
+            deleteQuery.executeUpdate();
+            return ok("Thumnail Failure Flag cleared for file id=: " + df.getId() + ".");
+        } catch (WrappedResponse r) {
+            logger.info("Could not find file with the id: " + fileId);
+            return error(Status.BAD_REQUEST, "Could not find file with the id: " + fileId);
+        }
+    }
+    
 }

From 63e98b3b60a4baae98f1f88a282b97694929c443 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 17 May 2023 14:16:47 -0400
Subject: [PATCH 137/922] make clearer

---
 .../java/edu/harvard/iq/dataverse/DataFileServiceBean.java  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index a5822828682..f41565c9449 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -1149,11 +1149,9 @@ public boolean isThumbnailAvailable (DataFile file) {
            file.setPreviewImageAvailable(true);
            this.save(file); 
            return true;
-       } else {
-           file.setPreviewsHaveFailed(true);
-           this.save(file);
        }
-
+       file.setPreviewsHaveFailed(true);
+       this.save(file);
        return false;
     }
 

From 2671cb75effb5425d02b3e874c7525b7833dc533 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 17 May 2023 14:25:58 -0400
Subject: [PATCH 138/922] update comment

---
 src/main/java/edu/harvard/iq/dataverse/DvObject.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
index 6cb3816e3f1..87619450133 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
@@ -182,8 +182,11 @@ public void setPreviewImageAvailable(boolean status) {
         this.previewImageAvailable = status;
     }
     
-    /** Indicates whether a previous attempt to generate a preview image has failed, regardless of size.
-     * If so, we won't want to try again every time the preview/thumbnail is requested for a view.
+    /**
+     * Indicates whether a previous attempt to generate a preview image has failed,
+     * regardless of size. This could be due to the file not being accessible, or a
+     * real failure in generating the thumbnail. In both cases, we won't want to try
+     * again every time the preview/thumbnail is requested for a view.
      */
     private boolean previewsHaveFailed;
 

From 19db99b1427700c9cc4ad462c0edd017e6dd5799 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 17 May 2023 14:26:28 -0400
Subject: [PATCH 139/922] remove setting flag where datafile is not clearly
 being saved to db

---
 .../harvard/iq/dataverse/dataaccess/ImageThumbConverter.java    | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index ab9294eea72..921faba7989 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -126,7 +126,6 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
             }
             if (!thumbnailGenerated) {
                 logger.fine("No thumbnail generated for " + file.getId());
-                file.setPreviewsHaveFailed(true);
             }
             return thumbnailGenerated;
         }
@@ -459,7 +458,6 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) {
                 if (!generated) {
                     // Record failure
                     logger.fine("Failed to generate base64 thumbnail for file id: " + file.getId());
-                    file.setPreviewsHaveFailed(true);
                 } else {
                     // Success - try to open again:
                     try {

From 156d025970eeb5223b6fd8343db09cafee057fed Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Thu, 1 Jun 2023 15:09:25 -0400
Subject: [PATCH 140/922] fix non-merge-able error when recording thumb fail

---
 .../iq/dataverse/DataFileServiceBean.java     | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index f41565c9449..880b2ea7dc4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -1142,17 +1142,17 @@ public boolean isThumbnailAvailable (DataFile file) {
          is more important... 
         
         */
-                
         
-       if (ImageThumbConverter.isThumbnailAvailable(file)) {
-           file = this.find(file.getId());
-           file.setPreviewImageAvailable(true);
-           this.save(file); 
-           return true;
-       }
-       file.setPreviewsHaveFailed(true);
-       this.save(file);
-       return false;
+        file = this.find(file.getId());
+        if (ImageThumbConverter.isThumbnailAvailable(file)) {
+            file.setPreviewImageAvailable(true);
+            this.save(file);
+            return true;
+        } else {
+            file.setPreviewsHaveFailed(true);
+            this.save(file);
+            return false;
+        }
     }
 
     

From 97aa46cb3e9bd2d424961e68e9d024216740c57f Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 13 Jun 2023 16:50:38 -0400
Subject: [PATCH 141/922] rename script

---
 ...humb-failures.sql => V5.13.0.2__9506-track-thumb-failures.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V5.13.0.1__9506-track-thumb-failures.sql => V5.13.0.2__9506-track-thumb-failures.sql} (100%)

diff --git a/src/main/resources/db/migration/V5.13.0.1__9506-track-thumb-failures.sql b/src/main/resources/db/migration/V5.13.0.2__9506-track-thumb-failures.sql
similarity index 100%
rename from src/main/resources/db/migration/V5.13.0.1__9506-track-thumb-failures.sql
rename to src/main/resources/db/migration/V5.13.0.2__9506-track-thumb-failures.sql

From dbc36c9d938571a5b61156611c445d266fbafe76 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 13 Jun 2023 17:06:19 -0400
Subject: [PATCH 142/922] refactor - remove duplicate code

---
 .../dataaccess/ImageThumbConverter.java       | 29 ++++++-------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index 921faba7989..fb0785ffd7b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -114,7 +114,11 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
             logger.fine("Found cached thumbnail for " + file.getId());
             return true;
         }
+        return generateThumbnail(storageIO, size);
 
+    }
+
+    private static boolean generateThumbnail(StorageIO<DataFile> storageIO, int size) {
         logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId());
         // Don't try to generate if there have been failures:
         if (!file.isPreviewsHaveFailed()) {
@@ -131,7 +135,6 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
         }
 
         return false;
-
     }
 
     // Note that this method works on ALL file types for which thumbnail 
@@ -446,25 +449,11 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) {
             logger.fine("Null channel for aux object " + THUMBNAIL_SUFFIX + size);
 
             // try to generate, if not available and hasn't failed before
-            logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + "to generate base64 thumbnail, file id: " + file.getId());
-            if (!file.isPreviewsHaveFailed()) {
-                boolean generated = false;
-                if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
-                    generated = generateImageThumbnail(storageIO, size);
-                } else if (file.getContentType().equalsIgnoreCase("application/pdf")) {
-                    generated = generatePDFThumbnail(storageIO, size);
-                }
-
-                if (!generated) {
-                    // Record failure
-                    logger.fine("Failed to generate base64 thumbnail for file id: " + file.getId());
-                } else {
-                    // Success - try to open again:
-                    try {
-                        cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size);
-                    } catch (Exception ioEx) {
-                        cachedThumbnailChannel = null;
-                    }
+            if(generateThumbnail(storageIO, size)) {
+                try {
+                    cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size);
+                } catch (Exception ioEx) {
+                    cachedThumbnailChannel = null;
                 }
             }
 

From 0c8972304a43c25ed1de1c5cc6cc1c09ef419948 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 14 Jun 2023 10:30:05 -0400
Subject: [PATCH 143/922] try ds logos as url requests

---
 .../iq/dataverse/ThumbnailServiceWrapper.java    | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index e2bb21c8a4c..66f79472178 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -5,6 +5,7 @@
  */
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.api.Datasets;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
@@ -12,7 +13,8 @@
 import static edu.harvard.iq.dataverse.dataset.DatasetUtil.datasetLogoThumbnail;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
 import edu.harvard.iq.dataverse.util.FileUtil;
-import java.io.File;
+import edu.harvard.iq.dataverse.util.SystemConfig;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -21,6 +23,8 @@
 import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.logging.Logger;
+
 import javax.ejb.EJB;
 import javax.enterprise.context.RequestScoped;
 import javax.faces.view.ViewScoped;
@@ -36,6 +40,9 @@
 @RequestScoped
 @Named
 public class ThumbnailServiceWrapper implements java.io.Serializable  {
+    
+    private static final Logger logger = Logger.getLogger(ThumbnailServiceWrapper.class.getCanonicalName());
+    
     @Inject
     PermissionsWrapper permissionsWrapper;
     @EJB
@@ -214,7 +221,13 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
             this.dvobjectThumbnailsMap.put(datasetId, "");
             return null; 
         }
+
+        String url = SystemConfig.getDataverseSiteUrlStatic() + "/datasets/" + dataset.getId() + "/logo";
+        logger.fine("getDatasetCardImageAsBase64Url: " + url);
+        this.dvobjectThumbnailsMap.put(datasetId,url);
+        return url;
         
+/*        
         String cardImageUrl = null;
         StorageIO<Dataset> dataAccess = null;
                 
@@ -320,6 +333,7 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
         //logger.info("dataset id " + result.getEntityId() + ", returning " + cardImageUrl);
 
         return cardImageUrl;
+        */
     }
     
     // it's the responsibility of the user - to make sure the search result

From dc4b6ae5201af228b1b484c6dd430713f8728ccc Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 14 Jun 2023 17:19:41 -0400
Subject: [PATCH 144/922] set the datasetid for search cards

---
 .../java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 66f79472178..4c3778527d7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -194,6 +194,7 @@ public String getDatasetCardImageAsBase64Url(SolrSearchResult result) {
             return null;
         }
         Dataset dataset = (Dataset)result.getEntity();
+        dataset.setId(result.getEntityId());
         
         Long versionId = result.getDatasetVersionId();
 

From 546cfdf2048158320e76a9345e9ebc3caf7ca6c2 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Thu, 1 Jun 2023 15:09:25 -0400
Subject: [PATCH 145/922] fix non-merge-able error when recording thumb fail

---
 .../java/edu/harvard/iq/dataverse/DataFileServiceBean.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 880b2ea7dc4..ec12480d28d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -1148,11 +1148,10 @@ public boolean isThumbnailAvailable (DataFile file) {
             file.setPreviewImageAvailable(true);
             this.save(file);
             return true;
-        } else {
-            file.setPreviewsHaveFailed(true);
-            this.save(file);
-            return false;
         }
+        file.setPreviewsHaveFailed(true);
+        this.save(file);
+        return false;
     }
 
     

From d3a48dffdfaa56bba065b3c36a2b6469e4227c33 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 14 Jun 2023 17:44:02 -0400
Subject: [PATCH 146/922] typo

---
 .../java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 4c3778527d7..8dda91fd6a3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -223,7 +223,7 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
             return null; 
         }
 
-        String url = SystemConfig.getDataverseSiteUrlStatic() + "/datasets/" + dataset.getId() + "/logo";
+        String url = SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/" + dataset.getId() + "/logo";
         logger.fine("getDatasetCardImageAsBase64Url: " + url);
         this.dvobjectThumbnailsMap.put(datasetId,url);
         return url;

From f505428f12a5ead774642837bdb871deda34ee27 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 19 Jun 2023 13:13:01 -0400
Subject: [PATCH 147/922] only send url if thumb should exist

---
 .../iq/dataverse/ThumbnailServiceWrapper.java     | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 8dda91fd6a3..19c53ffa77e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -8,6 +8,7 @@
 import edu.harvard.iq.dataverse.api.Datasets;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.dataset.DatasetUtil;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataset.DatasetUtil;
 import static edu.harvard.iq.dataverse.dataset.DatasetUtil.datasetLogoThumbnail;
@@ -222,6 +223,20 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
             this.dvobjectThumbnailsMap.put(datasetId, "");
             return null; 
         }
+        DataFile thumbnailFile = dataset.getThumbnailFile();
+
+        if (thumbnailFile == null) {
+                thumbnailFile = DatasetUtil.attemptToAutomaticallySelectThumbnailFromDataFiles(dataset, null);
+                if (thumbnailFile == null) {
+                    logger.fine("Dataset (id :" + dataset.getId() + ") does not have a logo available that could be selected automatically.");
+                    return null;
+                }
+        }
+        if (thumbnailFile.isRestricted()) {
+            logger.fine("Dataset (id :" + dataset.getId() + ") has a logo the user selected but the file must have later been restricted. Returning null.");
+            return null;
+        }
+        
 
         String url = SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/" + dataset.getId() + "/logo";
         logger.fine("getDatasetCardImageAsBase64Url: " + url);

From 2d177a60fe67df26bafad35cf237e048a21545ee Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 19 Jun 2023 15:08:15 -0400
Subject: [PATCH 148/922] use inputStream.transferTo

---
 .../dataaccess/ImageThumbConverter.java       | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index fb0785ffd7b..bd87c5541a5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -223,30 +223,32 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
         }
 
         if (tempFilesRequired) {
-            ReadableByteChannel pdfFileChannel;
-
+            //ReadableByteChannel pdfFileChannel;
+            InputStream inputStream = null; 
             try {
                 storageIO.open();
-                //inputStream = storageIO.getInputStream();
-                pdfFileChannel = storageIO.getReadChannel();
+                inputStream = storageIO.getInputStream();
+                //pdfFileChannel = storageIO.getReadChannel();
             } catch (Exception ioex) {
                 logger.warning("caught Exception trying to open an input stream for " + storageIO.getDataFile().getStorageIdentifier());
                 return false;
             }
 
             File tempFile;
-            FileChannel tempFileChannel = null;
+            OutputStream outputStream = null;
+            //FileChannel tempFileChannel = null;
             try {
                 tempFile = File.createTempFile("tempFileToRescale", ".tmp");
-                tempFileChannel = new FileOutputStream(tempFile).getChannel();
+                outputStream = new FileOutputStream(tempFile);
+                inputStream.transferTo(outputStream);
 
-                tempFileChannel.transferFrom(pdfFileChannel, 0, storageIO.getSize());
+                //tempFileChannel.transferFrom(pdfFileChannel, 0, storageIO.getSize());
             } catch (IOException ioex) {
                 logger.warning("GenerateImageThumb: failed to save pdf bytes in a temporary file.");
                 return false;
             } finally {
-                IOUtils.closeQuietly(tempFileChannel);
-                IOUtils.closeQuietly(pdfFileChannel);
+                IOUtils.closeQuietly(inputStream);
+                IOUtils.closeQuietly(outputStream);
             }
             sourcePdfFile = tempFile;
         }

From 6540b5da0966addffa3a0a6a9d7e67735f89e237 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 19 Jun 2023 15:42:29 -0400
Subject: [PATCH 149/922] add debug

---
 .../harvard/iq/dataverse/dataaccess/ImageThumbConverter.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index bd87c5541a5..4a2b8ea0e6d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -240,7 +240,8 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
             try {
                 tempFile = File.createTempFile("tempFileToRescale", ".tmp");
                 outputStream = new FileOutputStream(tempFile);
-                inputStream.transferTo(outputStream);
+                long sz = inputStream.transferTo(outputStream);
+                logger.info(" wrote " + sz + " bytes to " + tempFile.getAbsolutePath());
 
                 //tempFileChannel.transferFrom(pdfFileChannel, 0, storageIO.getSize());
             } catch (IOException ioex) {
@@ -763,7 +764,7 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size)
             try {
                 fileSize = new File(fileLocation).length();
             } catch (Exception ex) {
-                // 
+               logger.warning("Can't open file: " + fileLocation);
             }
 
             if (fileSize == 0 || fileSize > sizeLimit) {

From e202d0abc7395fe85218745510b32ade9b6ca770 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 19 Jun 2023 16:15:58 -0400
Subject: [PATCH 150/922] more debug

---
 .../iq/dataverse/dataaccess/ImageThumbConverter.java        | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index 4a2b8ea0e6d..3033269f3bc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -196,6 +196,7 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
         // We rely on ImageMagick to convert PDFs; so if it's not installed, 
         // better give up right away: 
         if (!isImageMagickInstalled()) {
+            logger.info("Couldn't find IM");
             return false;
         }
 
@@ -218,12 +219,15 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
             tempFilesRequired = true;
 
         } catch (IOException ioex) {
+            logger.warning(ioex.getMessage());
+            ioex.printStackTrace();
             // this on the other hand is likely a fatal condition :(
             return false;
         }
 
         if (tempFilesRequired) {
             //ReadableByteChannel pdfFileChannel;
+            logger.info("Creating temp file");
             InputStream inputStream = null; 
             try {
                 storageIO.open();
@@ -241,7 +245,7 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
                 tempFile = File.createTempFile("tempFileToRescale", ".tmp");
                 outputStream = new FileOutputStream(tempFile);
                 long sz = inputStream.transferTo(outputStream);
-                logger.info(" wrote " + sz + " bytes to " + tempFile.getAbsolutePath());
+                logger.info("Wrote " + sz + " bytes to " + tempFile.getAbsolutePath());
 
                 //tempFileChannel.transferFrom(pdfFileChannel, 0, storageIO.getSize());
             } catch (IOException ioex) {

From b9cd2bbf0c42fb4e7aada29d7cea817c195ca75d Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 20 Jun 2023 10:22:05 -0400
Subject: [PATCH 151/922] include failed preview flag in queries

---
 .../edu/harvard/iq/dataverse/DatasetVersionServiceBean.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 439e4b17ed4..0bd0a01aef1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -762,7 +762,7 @@ public Long getThumbnailByVersionId(Long versionId) {
                         + "AND df.id = o.id "
                         + "AND fm.datasetversion_id = dv.id "
                         + "AND fm.datafile_id = df.id "
-                        // + "AND o.previewImageAvailable = false "
+                        + "AND o.previewshavefailed = false "
                         + "AND df.restricted = false "
                         + "AND df.embargo_id is null "
                         + "AND df.contenttype LIKE 'image/%' "
@@ -796,7 +796,7 @@ public Long getThumbnailByVersionId(Long versionId) {
                         + "AND df.id = o.id "
                         + "AND fm.datasetversion_id = dv.id "
                         + "AND fm.datafile_id = df.id "
-                        // + "AND o.previewImageAvailable = false "
+                        + "AND o.previewshavefailed = false "
                         + "AND df.restricted = false "
                         + "AND df.embargo_id is null "
                         + "AND df.contenttype = 'application/pdf' "

From ac5a9564848ba241a993e8e9252641820e9041b4 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 20 Jun 2023 10:22:59 -0400
Subject: [PATCH 152/922] use getThumbnailByVersionId

---
 .../iq/dataverse/ThumbnailServiceWrapper.java   | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 19c53ffa77e..ff5e510e82c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -226,23 +226,20 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
         DataFile thumbnailFile = dataset.getThumbnailFile();
 
         if (thumbnailFile == null) {
-                thumbnailFile = DatasetUtil.attemptToAutomaticallySelectThumbnailFromDataFiles(dataset, null);
-                if (thumbnailFile == null) {
-                    logger.fine("Dataset (id :" + dataset.getId() + ") does not have a logo available that could be selected automatically.");
-                    return null;
-                }
-        }
-        if (thumbnailFile.isRestricted()) {
-            logger.fine("Dataset (id :" + dataset.getId() + ") has a logo the user selected but the file must have later been restricted. Returning null.");
-            return null;
+
+            // We attempt to auto-select via the optimized, native query-based method
+            // from the DatasetVersionService:
+            if (datasetVersionService.getThumbnailByVersionId(versionId) == null) {
+                return null;
+            }
         }
-        
 
         String url = SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/" + dataset.getId() + "/logo";
         logger.fine("getDatasetCardImageAsBase64Url: " + url);
         this.dvobjectThumbnailsMap.put(datasetId,url);
         return url;
         
+
 /*        
         String cardImageUrl = null;
         StorageIO<Dataset> dataAccess = null;

From 98acd6b50af770779329de1201663d8599edf16a Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 20 Jun 2023 10:49:24 -0400
Subject: [PATCH 153/922] cleanup

---
 .../dataverse/dataaccess/ImageThumbConverter.java   | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index 3033269f3bc..458b8da227b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -196,7 +196,7 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
         // We rely on ImageMagick to convert PDFs; so if it's not installed, 
         // better give up right away: 
         if (!isImageMagickInstalled()) {
-            logger.info("Couldn't find IM");
+            logger.fine("Couldn't find ImageMagick");
             return false;
         }
 
@@ -220,19 +220,15 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
 
         } catch (IOException ioex) {
             logger.warning(ioex.getMessage());
-            ioex.printStackTrace();
             // this on the other hand is likely a fatal condition :(
             return false;
         }
 
         if (tempFilesRequired) {
-            //ReadableByteChannel pdfFileChannel;
-            logger.info("Creating temp file");
             InputStream inputStream = null; 
             try {
                 storageIO.open();
                 inputStream = storageIO.getInputStream();
-                //pdfFileChannel = storageIO.getReadChannel();
             } catch (Exception ioex) {
                 logger.warning("caught Exception trying to open an input stream for " + storageIO.getDataFile().getStorageIdentifier());
                 return false;
@@ -240,14 +236,11 @@ private static boolean generatePDFThumbnail(StorageIO<DataFile> storageIO, int s
 
             File tempFile;
             OutputStream outputStream = null;
-            //FileChannel tempFileChannel = null;
             try {
                 tempFile = File.createTempFile("tempFileToRescale", ".tmp");
                 outputStream = new FileOutputStream(tempFile);
-                long sz = inputStream.transferTo(outputStream);
-                logger.info("Wrote " + sz + " bytes to " + tempFile.getAbsolutePath());
-
-                //tempFileChannel.transferFrom(pdfFileChannel, 0, storageIO.getSize());
+                //Reads/transfers all bytes from the input stream to the output stream. 
+                inputStream.transferTo(outputStream);
             } catch (IOException ioex) {
                 logger.warning("GenerateImageThumb: failed to save pdf bytes in a temporary file.");
                 return false;

From 98e5b3fbae8871ef0fecbd0550ad8fefb00e2b22 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 12:34:37 +0200
Subject: [PATCH 154/922] fix(ct): enable sane default for upload storage
 location in containers

The default from microprofile-config.properties does NOT work,
as the location must already be resolvable while the servlet is
being initialized - the app shipped defaults file is not yet read
at this point.

This is similar to the database options, which must be set using one
of the other Payara included config sources. (Non-easily resolvable
timing issue). The solution for containers is to add an env var to
the docker file, which can be overriden by any env var from compose
or K8s etc. (Problem is the high ordinal of the env source though)
---
 src/main/docker/Dockerfile                                 | 4 +++-
 src/main/resources/META-INF/microprofile-config.properties | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
index 88020a118b5..f64e88cb414 100644
--- a/src/main/docker/Dockerfile
+++ b/src/main/docker/Dockerfile
@@ -27,7 +27,9 @@ FROM $BASE_IMAGE
 # Make Payara use the "ct" profile for MicroProfile Config. Will switch various defaults for the application
 # setup in META-INF/microprofile-config.properties.
 # See also https://download.eclipse.org/microprofile/microprofile-config-3.0/microprofile-config-spec-3.0.html#configprofile
-ENV MP_CONFIG_PROFILE=ct
+ENV MP_CONFIG_PROFILE=ct \
+    # NOTE: this cannot be provided as default from microprofile-config.properties as not yet avail when servlet starts
+    DATAVERSE_FILES_UPLOADS="${STORAGE_DIR}/uploads"
 
 # Copy app and deps from assembly in proper layers
 COPY --chown=payara:payara maven/deps ${DEPLOY_DIR}/dataverse/WEB-INF/lib/
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 7c16495f870..748ed6de55a 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -12,7 +12,6 @@ dataverse.build=
 dataverse.files.directory=/tmp/dataverse
 # The variables are replaced with the environment variables from our base image, but still easy to override
 %ct.dataverse.files.directory=${STORAGE_DIR}
-%ct.dataverse.files.uploads=${STORAGE_DIR}/uploads
 
 # SEARCH INDEX
 dataverse.solr.host=localhost

From d71cdf2d427011fc660794bb12afbab9db1c2bc7 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 16:07:03 +0200
Subject: [PATCH 155/922] fix(ct,conf): switch to different approach to default
 upload location

Instead of trying to provide a default using STORAGE_DIR env var from
microprofile-config.properties as before, using this env var reference
in glassfish-web.xml directly now. By defaulting to "." if not present
(as in classic installations), it is fully equivalent to the former
hardcoded default value.

Providing a synced variant of it in microprofile-config.properties
and leaving a hint about the pitfalls, we can reuse the setting for
other purposes within the codebase as well (and expect the same
behaviour because same defaults).
---
 src/main/docker/Dockerfile                                 | 4 +---
 src/main/resources/META-INF/microprofile-config.properties | 6 ++++++
 src/main/webapp/WEB-INF/glassfish-web.xml                  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
index f64e88cb414..88020a118b5 100644
--- a/src/main/docker/Dockerfile
+++ b/src/main/docker/Dockerfile
@@ -27,9 +27,7 @@ FROM $BASE_IMAGE
 # Make Payara use the "ct" profile for MicroProfile Config. Will switch various defaults for the application
 # setup in META-INF/microprofile-config.properties.
 # See also https://download.eclipse.org/microprofile/microprofile-config-3.0/microprofile-config-spec-3.0.html#configprofile
-ENV MP_CONFIG_PROFILE=ct \
-    # NOTE: this cannot be provided as default from microprofile-config.properties as not yet avail when servlet starts
-    DATAVERSE_FILES_UPLOADS="${STORAGE_DIR}/uploads"
+ENV MP_CONFIG_PROFILE=ct
 
 # Copy app and deps from assembly in proper layers
 COPY --chown=payara:payara maven/deps ${DEPLOY_DIR}/dataverse/WEB-INF/lib/
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 748ed6de55a..f3745126cb2 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -12,6 +12,12 @@ dataverse.build=
 dataverse.files.directory=/tmp/dataverse
 # The variables are replaced with the environment variables from our base image, but still easy to override
 %ct.dataverse.files.directory=${STORAGE_DIR}
+# NOTE: the following uses STORAGE_DIR for both containers and classic installations. By defaulting to "." if not
+#       present, it equals the hardcoded default from before again. Also, be aware that this props file cannot provide
+#       any value for lookups in glassfish-web.xml during servlet initialization, as this file will not have
+#       been read yet! The names and their values are in sync here and over there to ensure the config checker
+#       is able to check for the directories (exist + writeable).
+dataverse.files.uploads=${STORAGE_DIR:.}/uploads
 
 # SEARCH INDEX
 dataverse.solr.host=localhost
diff --git a/src/main/webapp/WEB-INF/glassfish-web.xml b/src/main/webapp/WEB-INF/glassfish-web.xml
index e56d7013abf..8041ebd4447 100644
--- a/src/main/webapp/WEB-INF/glassfish-web.xml
+++ b/src/main/webapp/WEB-INF/glassfish-web.xml
@@ -18,5 +18,5 @@
     This folder is not only holding compiled JSP pages but also the place where file streams are stored
     during uploads. As Dataverse does not use any JSP, there will only be uploads stored here.
   -->
-  <property name="tempdir" value="${MPCONFIG=dataverse.files.uploads:./uploads}"/>
+  <property name="tempdir" value="${MPCONFIG=dataverse.files.uploads:${ENV=STORAGE_DIR:.}/uploads}"/>
 </glassfish-web-app>

From a4ec3a66e76aa1559aea0c05cedc2da2b38d7b03 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 16:44:08 +0200
Subject: [PATCH 156/922] feat(conf): introduce ConfigCheckService to validate
 config on startup #9572

Starting with important local storage locations for the Dataverse application,
this service uses EJB startup mechanisms to verify configuration bits on startup.

Checks for the temp storage location and JSF upload location as crucial parts
of the app, which, if not exist or write protected, while only cause errors and
failures on the first data upload attempt. This is not desirable as it might cause
users to be blocked.
---
 .../settings/ConfigCheckService.java          | 65 +++++++++++++++++++
 .../iq/dataverse/settings/JvmSettings.java    |  1 +
 .../harvard/iq/dataverse/util/FileUtil.java   | 29 ++++-----
 3 files changed, 77 insertions(+), 18 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
new file mode 100644
index 00000000000..4ba028903b0
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
@@ -0,0 +1,65 @@
+package edu.harvard.iq.dataverse.settings;
+
+import edu.harvard.iq.dataverse.util.FileUtil;
+
+import javax.annotation.PostConstruct;
+import javax.ejb.DependsOn;
+import javax.ejb.Singleton;
+import javax.ejb.Startup;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+@Startup
+@Singleton
+@DependsOn("StartupFlywayMigrator")
+public class ConfigCheckService {
+    
+    private static final Logger logger = Logger.getLogger(ConfigCheckService.class.getCanonicalName());
+
+    public static class ConfigurationError extends RuntimeException {
+        public ConfigurationError(String message) {
+            super(message);
+        }
+    }
+    
+    @PostConstruct
+    public void startup() {
+        if (!checkSystemDirectories()) {
+            throw new ConfigurationError("Not all configuration checks passed successfully. See logs above.");
+        }
+    }
+    
+    /**
+     * In this method, we check the existence and write-ability of all important directories we use during
+     * normal operations. It does not include checks for the storage system. If directories are not available,
+     * try to create them (and fail when not allowed to).
+     *
+     * @return True if all checks successful, false otherwise.
+     */
+    public boolean checkSystemDirectories() {
+        Map<Path, String> paths = Map.of(
+                Path.of(JvmSettings.UPLOADS_DIRECTORY.lookup()), "temporary JSF upload space (see " + JvmSettings.UPLOADS_DIRECTORY.getScopedKey() + ")",
+                Path.of(FileUtil.getFilesTempDirectory()), "temporary processing space (see " + JvmSettings.FILES_DIRECTORY.getScopedKey() + ")");
+        
+        boolean success = true;
+        for (Path path : paths.keySet()) {
+            if (Files.notExists(path)) {
+                try {
+                    Files.createDirectories(path);
+                } catch (IOException e) {
+                    logger.log(Level.SEVERE, () -> "Could not create directory " + path + " for " + paths.get(path));
+                    success = false;
+                }
+            } else if (!Files.isWritable(path)) {
+                logger.log(Level.SEVERE, () -> "Directory " + path + " for " + paths.get(path) + " exists, but is not writeable");
+                success = false;
+            }
+        }
+        return success;
+    }
+
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index ff04a633ea7..c5c5682821a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -49,6 +49,7 @@ public enum JvmSettings {
     // FILES SETTINGS
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
+    UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 6bb7e1d583b..ee1ee5a6a1c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -40,6 +40,7 @@
 import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
 import edu.harvard.iq.dataverse.ingest.IngestableDataChecker;
 import edu.harvard.iq.dataverse.license.License;
+import edu.harvard.iq.dataverse.settings.ConfigCheckService;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.file.BagItFileHandler;
 import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;
@@ -1478,25 +1479,17 @@ public static boolean canIngestAsTabular(String mimeType) {
         }
     }
     
+    /**
+     * Return the location where data should be stored temporarily after uploading (UI or API)
+     * for local processing (ingest, unzip, ...) and transfer to final destination (see storage subsystem).
+     *
+     * This location is checked to be configured, does exist, and is writeable via
+     * {@link ConfigCheckService#checkSystemDirectories()}.
+     *
+     * @return String with a path to the temporary location. Will not be null (former versions did to indicate failure)
+     */
     public static String getFilesTempDirectory() {
-        
-        String filesRootDirectory = JvmSettings.FILES_DIRECTORY.lookup();
-        String filesTempDirectory = filesRootDirectory + "/temp";
-
-        if (!Files.exists(Paths.get(filesTempDirectory))) {
-            /* Note that "createDirectories()" must be used - not 
-             * "createDirectory()", to make sure all the parent 
-             * directories that may not yet exist are created as well. 
-             */
-            try {
-                Files.createDirectories(Paths.get(filesTempDirectory));
-            } catch (IOException ex) {
-                logger.severe("Failed to create filesTempDirectory: " + filesTempDirectory );
-                return null;
-            }
-        }
-
-        return filesTempDirectory;
+        return JvmSettings.FILES_DIRECTORY.lookup() + File.separator + "temp";
     }
     
     public static void generateS3PackageStorageIdentifier(DataFile dataFile) {

From 6999093dcea8e889a24aafbe84dd6035e8a4b5db Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 17:37:40 +0200
Subject: [PATCH 157/922] feat(conf): make docroot location configurable #9662

Add JVM Setting and add to config checker on startup to ensure target location is in good shape.
---
 .../harvard/iq/dataverse/settings/ConfigCheckService.java | 3 ++-
 .../edu/harvard/iq/dataverse/settings/JvmSettings.java    | 1 +
 .../resources/META-INF/microprofile-config.properties     | 1 +
 src/main/webapp/WEB-INF/glassfish-web.xml                 | 8 ++++----
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
index 4ba028903b0..443d12fc17a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
@@ -43,7 +43,8 @@ public void startup() {
     public boolean checkSystemDirectories() {
         Map<Path, String> paths = Map.of(
                 Path.of(JvmSettings.UPLOADS_DIRECTORY.lookup()), "temporary JSF upload space (see " + JvmSettings.UPLOADS_DIRECTORY.getScopedKey() + ")",
-                Path.of(FileUtil.getFilesTempDirectory()), "temporary processing space (see " + JvmSettings.FILES_DIRECTORY.getScopedKey() + ")");
+                Path.of(FileUtil.getFilesTempDirectory()), "temporary processing space (see " + JvmSettings.FILES_DIRECTORY.getScopedKey() + ")",
+                Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup()), "docroot space (see " + JvmSettings.DOCROOT_DIRECTORY.getScopedKey() + ")");
         
         boolean success = true;
         for (Path path : paths.keySet()) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index c5c5682821a..540dc8201a0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -50,6 +50,7 @@ public enum JvmSettings {
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
     UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"),
+    DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index f3745126cb2..597d50b2e0c 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -18,6 +18,7 @@ dataverse.files.directory=/tmp/dataverse
 #       been read yet! The names and their values are in sync here and over there to ensure the config checker
 #       is able to check for the directories (exist + writeable).
 dataverse.files.uploads=${STORAGE_DIR:.}/uploads
+dataverse.files.docroot=${STORAGE_DIR:.}/docroot
 
 # SEARCH INDEX
 dataverse.solr.host=localhost
diff --git a/src/main/webapp/WEB-INF/glassfish-web.xml b/src/main/webapp/WEB-INF/glassfish-web.xml
index 8041ebd4447..5088e5a7fba 100644
--- a/src/main/webapp/WEB-INF/glassfish-web.xml
+++ b/src/main/webapp/WEB-INF/glassfish-web.xml
@@ -10,10 +10,10 @@
   </jsp-config>
   <parameter-encoding default-charset="UTF-8"/>
   <!-- Find a list of properties here: https://docs.oracle.com/cd/E19798-01/821-1750/beayb/index.html -->
-  <property name="alternatedocroot_1" value="from=/guides/* dir=./docroot"/>
-  <property name="alternatedocroot_2" value="from=/dataexplore/* dir=./docroot"/>
-  <property name="alternatedocroot_logos" value="from=/logos/* dir=./docroot"/>
-  <property name="alternatedocroot_sitemap" value="from=/sitemap/* dir=./docroot"/>
+  <property name="alternatedocroot_1" value="from=/guides/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
+  <property name="alternatedocroot_2" value="from=/dataexplore/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
+  <property name="alternatedocroot_logos" value="from=/logos/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
+  <property name="alternatedocroot_sitemap" value="from=/sitemap/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
   <!--
     This folder is not only holding compiled JSP pages but also the place where file streams are stored
     during uploads. As Dataverse does not use any JSP, there will only be uploads stored here.

From 0e29e3e56ceae95abdf55bdf781a20b5fb5b886f Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 17:54:27 +0200
Subject: [PATCH 158/922] feat(conf): add some more details to error output if
 available in configuration checking

---
 .../iq/dataverse/settings/ConfigCheckService.java      | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
index 443d12fc17a..9e2a82d6b58 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
@@ -7,6 +7,7 @@
 import javax.ejb.Singleton;
 import javax.ejb.Startup;
 import java.io.IOException;
+import java.nio.file.FileSystemException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Map;
@@ -52,7 +53,14 @@ public boolean checkSystemDirectories() {
                 try {
                     Files.createDirectories(path);
                 } catch (IOException e) {
-                    logger.log(Level.SEVERE, () -> "Could not create directory " + path + " for " + paths.get(path));
+                    String details;
+                    if (e instanceof FileSystemException) {
+                        details = ": " + e.getClass();
+                    } else {
+                        details = "";
+                    }
+                    
+                    logger.log(Level.SEVERE, () -> "Could not create directory " + path + " for " + paths.get(path) + details);
                     success = false;
                 }
             } else if (!Files.isWritable(path)) {

From 4aac8f137199c13af9012db4e6c582f62439bf9e Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 18:04:13 +0200
Subject: [PATCH 159/922] fix(ct): make sure file permissions on app data
 volume when using compose

---
 docker-compose-dev.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 06277d9541f..d11ff0e5671 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -23,6 +23,7 @@ services:
     depends_on:
       - dev_postgres
       - dev_solr
+      - dev_dv_initializer
     volumes:
       - ./docker-dev-volumes/app/data:/dv
       - ./docker-dev-volumes/app/secrets:/secrets
@@ -43,6 +44,17 @@ services:
     networks:
       - dataverse
 
+  dev_dv_initializer:
+    container_name: "dev_dv_initializer"
+    image: gdcc/configbaker:unstable
+    restart: "no"
+    command:
+      - sh
+      - -c
+      - "fix-fs-perms.sh dv"
+    volumes:
+      - ./docker-dev-volumes/app/data:/dv
+
   dev_postgres:
     container_name: "dev_postgres"
     hostname: postgres

From 1c256e9e16868a705a495118072374bd0066537f Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 18:27:21 +0200
Subject: [PATCH 160/922] refactor(conf): make docroot and uploads locations
 more precise and document more of the behaviour

---
 .../META-INF/microprofile-config.properties       | 15 ++++++++-------
 src/main/webapp/WEB-INF/glassfish-web.xml         |  1 +
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 597d50b2e0c..b58c728316b 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -12,13 +12,14 @@ dataverse.build=
 dataverse.files.directory=/tmp/dataverse
 # The variables are replaced with the environment variables from our base image, but still easy to override
 %ct.dataverse.files.directory=${STORAGE_DIR}
-# NOTE: the following uses STORAGE_DIR for both containers and classic installations. By defaulting to "." if not
-#       present, it equals the hardcoded default from before again. Also, be aware that this props file cannot provide
-#       any value for lookups in glassfish-web.xml during servlet initialization, as this file will not have
-#       been read yet! The names and their values are in sync here and over there to ensure the config checker
-#       is able to check for the directories (exist + writeable).
-dataverse.files.uploads=${STORAGE_DIR:.}/uploads
-dataverse.files.docroot=${STORAGE_DIR:.}/docroot
+# NOTE: the following uses STORAGE_DIR for both containers and classic installations. By defaulting to
+#       "com.sun.aas.instanceRoot" if not present, it equals the hardcoded former default "." in glassfish-web.xml
+#       (which is relative to the domain root folder).
+#       Also, be aware that this props file cannot provide any value for lookups in glassfish-web.xml during servlet
+#       initialization, as this file will not have been read yet! The names and their values are in sync here and over
+#       there to ensure the config checker is able to check for the directories (exist + writeable).
+dataverse.files.uploads=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/uploads
+dataverse.files.docroot=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/docroot
 
 # SEARCH INDEX
 dataverse.solr.host=localhost
diff --git a/src/main/webapp/WEB-INF/glassfish-web.xml b/src/main/webapp/WEB-INF/glassfish-web.xml
index 5088e5a7fba..9677bf089e2 100644
--- a/src/main/webapp/WEB-INF/glassfish-web.xml
+++ b/src/main/webapp/WEB-INF/glassfish-web.xml
@@ -10,6 +10,7 @@
   </jsp-config>
   <parameter-encoding default-charset="UTF-8"/>
   <!-- Find a list of properties here: https://docs.oracle.com/cd/E19798-01/821-1750/beayb/index.html -->
+  <!-- NOTE: relative paths given will be relative to com.sun.aas.instanceRoot (usually domain1 folder) -->
   <property name="alternatedocroot_1" value="from=/guides/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
   <property name="alternatedocroot_2" value="from=/dataexplore/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
   <property name="alternatedocroot_logos" value="from=/logos/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>

From 2913a52f35645621bace35c93a9c0b2707004da1 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 21 Jun 2023 18:32:55 +0200
Subject: [PATCH 161/922] refactor(conf): simplify sitemap output location
 lookup using new docroot setting

---
 .../iq/dataverse/sitemap/SiteMapUtil.java     | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java
index e32b811ee2c..86ae697f771 100644
--- a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java
@@ -3,6 +3,8 @@
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObjectContainer;
+import edu.harvard.iq.dataverse.settings.ConfigCheckService;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.xml.XmlValidator;
 import java.io.File;
@@ -210,16 +212,17 @@ public static boolean stageFileExists() {
         }
         return false;
     }
-
+    
+    /**
+     * Lookup the location where to generate the sitemap.
+     *
+     * Note: the location is checked to be configured, does exist and is writeable in
+     * {@link ConfigCheckService#checkSystemDirectories()}
+     *
+     * @return Sitemap storage location ([docroot]/sitemap)
+     */
     private static String getSitemapPathString() {
-        String sitemapPathString = "/tmp";
-        // i.e. /usr/local/glassfish4/glassfish/domains/domain1
-        String domainRoot = System.getProperty("com.sun.aas.instanceRoot");
-        if (domainRoot != null) {
-            // Note that we write to a directory called "sitemap" but we serve just "/sitemap.xml" using PrettyFaces.
-            sitemapPathString = domainRoot + File.separator + "docroot" + File.separator + "sitemap";
-        }
-        return sitemapPathString;
+        return JvmSettings.DOCROOT_DIRECTORY.lookup() + File.separator + "sitemap";
 
     }
 }

From 610c65dc9ddd403041ee95475810db2977e57623 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 21 Jun 2023 12:56:13 -0400
Subject: [PATCH 162/922] rename and cleanup

---
 .../edu/harvard/iq/dataverse/DatasetPage.java |   2 +-
 .../iq/dataverse/DataverseServiceBean.java    |  45 -------
 .../iq/dataverse/ThumbnailServiceWrapper.java | 117 +-----------------
 .../search/SearchIncludeFragment.java         |   2 +-
 4 files changed, 6 insertions(+), 160 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 3d608153ba3..2ca1fb825f5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -483,7 +483,7 @@ public String getThumbnailString() {
 
             thumbnailString = datasetThumbnail.getBase64image();
         } else {
-            thumbnailString = thumbnailServiceWrapper.getDatasetCardImageAsBase64Url(dataset,
+            thumbnailString = thumbnailServiceWrapper.getDatasetCardImageAsUrl(dataset,
                     workingVersion.getId(),
                     !workingVersion.isDraft(),
                     ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE);
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index e092f209acd..e99458fbc9d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -346,51 +346,6 @@ public String getDataverseLogoThumbnailAsBase64ById(Long dvId) {
         } 
         return null;
     }
-    
-    /*
-    public boolean isDataverseLogoThumbnailAvailable(Dataverse dataverse, User user) {    
-        if (dataverse == null) {
-            return false; 
-        }
-                
-        // First, check if the dataverse has a defined logo: 
-        
-        //if (dataverse.getDataverseTheme() != null && dataverse.getDataverseTheme().getLogo() != null && !dataverse.getDataverseTheme().getLogo().equals("")) {
-            File dataverseLogoFile = getLogo(dataverse);
-            if (dataverseLogoFile != null) {
-                String logoThumbNailPath = null;
-
-                if (dataverseLogoFile.exists()) {
-                    logoThumbNailPath = ImageThumbConverter.generateImageThumbnailFromFile(dataverseLogoFile.getAbsolutePath(), 48);
-                    if (logoThumbNailPath != null) {
-                        return true;
-                    }
-                }
-            }
-        //}
-        */
-        // If there's no uploaded logo for this dataverse, go through its 
-        // [released] datasets and see if any of them have card images:
-        // 
-        // TODO:
-        // Discuss/Decide if we really want to do this - i.e., go through every
-        // file in every dataset below... 
-        // -- L.A. 4.0 beta14
-        /*
-        for (Dataset dataset : datasetService.findPublishedByOwnerId(dataverse.getId())) {
-            if (dataset != null) {
-                DatasetVersion releasedVersion = dataset.getReleasedVersion();
-                
-                if (releasedVersion != null) {
-                    if (datasetService.isDatasetCardImageAvailable(releasedVersion, user)) {
-                        return true;
-                    }
-                }
-            }
-        }   */     
-        /*
-        return false; 
-    } */
         
     private File getLogo(Dataverse dataverse) {
         if (dataverse.getId() == null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index ff5e510e82c..c75c29ea094 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -177,7 +177,7 @@ public boolean isThumbnailAvailable(DataFile entity) {
 
     // it's the responsibility of the user - to make sure the search result
     // passed to this method is of the Dataset type!
-    public String getDatasetCardImageAsBase64Url(SolrSearchResult result) {
+    public String getDatasetCardImageAsUrl(SolrSearchResult result) {
         // Before we do anything else, check if it's a harvested dataset; 
         // no need to check anything else if so (harvested datasets never have 
         // thumbnails)
@@ -199,10 +199,10 @@ public String getDatasetCardImageAsBase64Url(SolrSearchResult result) {
         
         Long versionId = result.getDatasetVersionId();
 
-        return getDatasetCardImageAsBase64Url(dataset, versionId, result.isPublishedState(), ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
+        return getDatasetCardImageAsUrl(dataset, versionId, result.isPublishedState(), ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
     }
     
-    public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, boolean autoselect, int size) {
+    public String getDatasetCardImageAsUrl(Dataset dataset, Long versionId, boolean autoselect, int size) {
         Long datasetId = dataset.getId();
         if (datasetId != null) {
             if (this.dvobjectThumbnailsMap.containsKey(datasetId)) {
@@ -235,118 +235,9 @@ public String getDatasetCardImageAsBase64Url(Dataset dataset, Long versionId, bo
         }
 
         String url = SystemConfig.getDataverseSiteUrlStatic() + "/api/datasets/" + dataset.getId() + "/logo";
-        logger.fine("getDatasetCardImageAsBase64Url: " + url);
+        logger.fine("getDatasetCardImageAsUrl: " + url);
         this.dvobjectThumbnailsMap.put(datasetId,url);
         return url;
-        
-
-/*        
-        String cardImageUrl = null;
-        StorageIO<Dataset> dataAccess = null;
-                
-        try{
-            dataAccess = DataAccess.getStorageIO(dataset);
-        }
-        catch(IOException ioex){
-          // ignore
-        }
-        
-        InputStream in = null;
-        // See if the dataset already has a dedicated thumbnail ("logo") saved as
-        // an auxilary file on the dataset level: 
-        // (don't bother checking if it exists; just try to open the input stream)
-        try {
-                in = dataAccess.getAuxFileAsInputStream(datasetLogoThumbnail + ".thumb" + size);
-                        //thumb48addedByImageThumbConverter);
-        } catch (Exception ioex) {
-              //ignore
-        }
-        
-        if (in != null) {
-            try {
-                byte[] bytes = IOUtils.toByteArray(in);
-                String base64image = Base64.getEncoder().encodeToString(bytes);
-                cardImageUrl = FileUtil.DATA_URI_SCHEME + base64image;
-                this.dvobjectThumbnailsMap.put(datasetId, cardImageUrl);
-                return cardImageUrl;
-            } catch (IOException ex) {
-                this.dvobjectThumbnailsMap.put(datasetId, "");
-                return null; 
-                // (alternatively, we could ignore the exception, and proceed with the 
-                // regular process of selecting the thumbnail from the available 
-                // image files - ?)
-            } finally
-	    {
-		    IOUtils.closeQuietly(in);
-	    }
-        } 
-
-        // If not, see if the dataset has one of its image files already assigned
-        // to be the designated thumbnail:
-        cardImageUrl = this.getAssignedDatasetImage(dataset, size);
-
-        if (cardImageUrl != null) {
-            //logger.info("dataset id " + result.getEntity().getId() + " has a dedicated image assigned; returning " + cardImageUrl);
-            return cardImageUrl;
-        }
-        
-        // And finally, try to auto-select the thumbnail (unless instructed not to):
-        
-        if (!autoselect) {
-            return null;
-        }
-
-        // We attempt to auto-select via the optimized, native query-based method 
-        // from the DatasetVersionService:
-        Long thumbnailImageFileId = datasetVersionService.getThumbnailByVersionId(versionId);
-
-        if (thumbnailImageFileId != null) {
-            //cardImageUrl = FILE_CARD_IMAGE_URL + thumbnailImageFileId;
-            if (this.dvobjectThumbnailsMap.containsKey(thumbnailImageFileId)) {
-                // Yes, return previous answer
-                //logger.info("using cached result for ... "+datasetId);
-                if (!"".equals(this.dvobjectThumbnailsMap.get(thumbnailImageFileId))) {
-                    return this.dvobjectThumbnailsMap.get(thumbnailImageFileId);
-                }
-                return null;
-            }
-
-            DataFile thumbnailImageFile = null;
-
-            if (dvobjectViewMap.containsKey(thumbnailImageFileId)
-                    && dvobjectViewMap.get(thumbnailImageFileId).isInstanceofDataFile()) {
-                thumbnailImageFile = (DataFile) dvobjectViewMap.get(thumbnailImageFileId);
-            } else {
-                thumbnailImageFile = dataFileService.findCheapAndEasy(thumbnailImageFileId);
-                if (thumbnailImageFile != null) {
-                    // TODO:
-                    // do we need this file on the map? - it may not even produce
-                    // a thumbnail!
-                    dvobjectViewMap.put(thumbnailImageFileId, thumbnailImageFile);
-                } else {
-                    this.dvobjectThumbnailsMap.put(thumbnailImageFileId, "");
-                    return null;
-                }
-            }
-
-            if (isThumbnailAvailable(thumbnailImageFile)) {
-                cardImageUrl = ImageThumbConverter.getImageThumbnailAsBase64(
-                        thumbnailImageFile,
-                        size);
-                        //ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
-            }
-
-            if (cardImageUrl != null) {
-                this.dvobjectThumbnailsMap.put(thumbnailImageFileId, cardImageUrl);
-            } else {
-                this.dvobjectThumbnailsMap.put(thumbnailImageFileId, "");
-            }
-        }
-
-        //logger.info("dataset id " + result.getEntityId() + ", returning " + cardImageUrl);
-
-        return cardImageUrl;
-        */
     }
     
     // it's the responsibility of the user - to make sure the search result
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index bfe397cf48c..99fe4cd979b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -1302,7 +1302,7 @@ public void setDisplayCardValues() {
                 result.setImageUrl(thumbnailServiceWrapper.getDataverseCardImageAsBase64Url(result));
             } else if (result.getType().equals("datasets")) {
                 if (result.getEntity() != null) {
-                    result.setImageUrl(thumbnailServiceWrapper.getDatasetCardImageAsBase64Url(result));
+                    result.setImageUrl(thumbnailServiceWrapper.getDatasetCardImageAsUrl(result));
                 }
                 
                 if (result.isHarvested()) {

From 391504de43d8992e4b97d506fdfc763e512a8fc4 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 21 Jun 2023 13:46:35 -0400
Subject: [PATCH 163/922] api docs

---
 doc/sphinx-guides/source/api/native-api.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index b39cf91337a..24f6c0d4ced 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -4649,3 +4649,23 @@ A curl example using an ``ID``
   curl -X POST -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/feedback
 
 Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`.
+
+.. _thumbnail_reset:
+
+Reset Thumbnail Failure Flags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If Dataverse attempts to create a thumbnail image for an image or pdf file and the attempt fails, Dataverse will set a flag for the file to avoid repeated attempts to generate the thumbnail.
+For cases where the problem may have been temporary (or fixed in a later Dataverse release), two API calls exist to reset this flag for all files or for a given file.
+
+Curl examples
+
+.. code-block:: bash
+
+  export SERVER_URL=http://localhost
+  export fileID=1234
+
+  curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag
+  
+  curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag/$fileID
+

From de7963a0635646f6c00e1362fc87152029394839 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 21 Jun 2023 13:53:30 -0400
Subject: [PATCH 164/922] refactor typo

---
 .../iq/dataverse/dataaccess/ImageThumbConverter.java        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index 458b8da227b..febf659b71a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -114,11 +114,11 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
             logger.fine("Found cached thumbnail for " + file.getId());
             return true;
         }
-        return generateThumbnail(storageIO, size);
+        return generateThumbnail(file, storageIO, size);
 
     }
 
-    private static boolean generateThumbnail(StorageIO<DataFile> storageIO, int size) {
+    private static boolean generateThumbnail(DataFile file, StorageIO<DataFile> storageIO, int size) {
         logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId());
         // Don't try to generate if there have been failures:
         if (!file.isPreviewsHaveFailed()) {
@@ -449,7 +449,7 @@ public static String getImageThumbnailAsBase64(DataFile file, int size) {
             logger.fine("Null channel for aux object " + THUMBNAIL_SUFFIX + size);
 
             // try to generate, if not available and hasn't failed before
-            if(generateThumbnail(storageIO, size)) {
+            if(generateThumbnail(file, storageIO, size)) {
                 try {
                     cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size);
                 } catch (Exception ioEx) {

From fd4d9199fade70e6a2387ad02051bbf4865fa9f2 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 22 Jun 2023 10:33:37 -0400
Subject: [PATCH 165/922] add gb fragment

---
 .../guestbook-terms-popup-fragment.xhtml      | 105 ++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 9c3391ef9ae..829fa6539b9 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -40,6 +40,111 @@
                     </div>
                 </div>
             </div>
+            <p:fragment rendered="#{workingVersion.dataset.guestbook != null and workingVersion.dataset.guestbook.enabled and guestbookAndTermsPopupRequired}" id="guestbookUIFragment">
+                <!--
+                Only validate the active context. For example, the user might be filling out
+                the guestbook on the Preview tab rather than after clicking the "download file"
+                button, even though the form is the same and on the same page. Without this boolean
+                in place, invalid fields on the other form (the non-active form) are checked, leading
+                to a validation failure.
+                -->
+                <ui:param name="validateThisContext" value="DO_GB_VALIDATION_#{popupContext}"/>
+                <div class="form-group">
+                    <label class="col-sm-3 control-label" for="guestbookuser_nameText">
+                            #{bundle.name}
+                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.nameRequired}" />
+                    </label>
+                    <div class="col-sm-6">
+                        <p:inputText id="guestbookuser_nameText"
+                                     required="#{param[validateThisContext] and workingVersion.dataset.guestbook.nameRequired}"
+                                     styleClass="form-control" value="#{guestbookResponse.name}"
+                                     requiredMessage="#{bundle['requiredField']}"
+                                     >
+                        </p:inputText>
+                        <p:message id="nameMessages" for="guestbookuser_nameText" display="text"/>
+                    </div>
+                </div>
+                <div class="form-group">
+                    <label class="col-sm-3 control-label" for="guestbookuser_email">
+                        #{bundle.email}
+                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.emailRequired}" />
+                    </label>
+                    <div class="col-sm-6">
+                        <p:inputText id="guestbookuser_email" required="#{param[validateThisContext] and workingVersion.dataset.guestbook.emailRequired}"
+                                     styleClass="form-control" value="#{guestbookResponse.email}"
+                                     requiredMessage="#{bundle['requiredField']}"
+                                     >
+                        </p:inputText>  
+                        <p:message id="emailMessages" for="guestbookuser_email" display="text"/>
+                    </div>
+                </div>
+                <div class="form-group">
+                    <label class="col-sm-3 control-label" for="guestbookuser_institution">
+                        #{bundle.institution}
+                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.institutionRequired}" />
+                    </label>
+                    <div class="col-sm-6">
+                        <p:inputText id="guestbookuser_institution" required="#{param[validateThisContext] and workingVersion.dataset.guestbook.institutionRequired}"
+                                     styleClass="form-control" value="#{guestbookResponse.institution}"
+                                     requiredMessage="#{bundle['requiredField']}"
+                                     >
+                        </p:inputText>
+                        <p:message id="institutionMessages" for="guestbookuser_institution" display="text"/>
+                    </div>
+                </div>
+                <div class="form-group">
+                    <label class="col-sm-3 control-label" for="guestbookuser_position">
+                        #{bundle.position}
+                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.positionRequired}" />
+                    </label>
+                    <div class="col-sm-6">
+                        <p:inputText id="guestbookuser_position"  required="#{param[validateThisContext] and workingVersion.dataset.guestbook.positionRequired}"
+                                     styleClass="form-control" value="#{guestbookResponse.position}"
+                                     requiredMessage="#{bundle['requiredField']}"
+                                     >
+                        </p:inputText>
+                        <p:message id="positionMessages" for="guestbookuser_position" display="text"/>
+                    </div>
+                </div>
+                <div class="form-group" jsf:rendered="#{!empty workingVersion.dataset.guestbook.customQuestions}">
+                    <label class="col-sm-3 control-label">
+                        #{bundle['dataset.guestbookResponse.guestbook.additionalQuestions']}
+                    </label>
+                    <div class="col-sm-6">
+                        <ui:repeat value="#{guestbookResponse.customQuestionResponses}" var="customQuestionResponse">
+                            <div class="text-left">
+                                <label class="control-label">
+                                    <h:outputText value="#{customQuestionResponse.customQuestion.questionString} "/>
+                                    <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{customQuestionResponse.customQuestion.required}" />
+                                </label>
+                                <p:inputText id="customQuestionResponse"
+                                             styleClass="form-control" value="#{customQuestionResponse.response}"
+                                             required="#{param[validateThisContext] and customQuestionResponse.customQuestion.required}"
+                                             rendered="#{customQuestionResponse.customQuestion.questionType=='text'}"
+                                             requiredMessage="#{bundle['requiredField']}">
+                                </p:inputText>
+                                <p:message id="cqMessages" for="customQuestionResponse" display="text"/>
+                                <p:inputTextarea id="customQuestionResponseTextArea" rows="8" autoResize="true"
+                                             styleClass="form-control"
+                                             value="#{customQuestionResponse.response}"
+                                             required="#{param['DO_GB_VALIDATION'] and customQuestionResponse.customQuestion.required}"
+                                             rendered="#{customQuestionResponse.customQuestion.questionType=='textarea'}"
+                                             requiredMessage="#{bundle['requiredField']}"/>
+                                <p:message id="cqMessagesTA" for="customQuestionResponseTextArea" display="text"/>
+                                <p:selectOneMenu id="customQuestionResponseSelect"
+                                                 styleClass="form-control" value="#{customQuestionResponse.response}"
+                                                 required="#{param[validateThisContext] and customQuestionResponse.customQuestion.required}"
+                                                 rendered="#{customQuestionResponse.customQuestion.questionType=='options'}"
+                                                 requiredMessage="#{bundle['requiredField']}">
+                                    <f:selectItem itemLabel="#{bundle.select}" itemValue="" noSelectionOption="true" />
+                                    <f:selectItems value="#{customQuestionResponse.responseSelectItems}" />
+                                </p:selectOneMenu>
+                                <p:message id="cqrsMessages" for="customQuestionResponseSelect" display="text"/>
+                            </div>
+                        </ui:repeat>
+                    </div>
+                </div>
+            </p:fragment>
         </div>
         <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle.download}">
             <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" 

From fc6a5e01c36a7fa172d7ba46a3cf2a39647bb808 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 22 Jun 2023 10:34:11 -0400
Subject: [PATCH 166/922] change render param - not clear this fragment stays
 here though

---
 src/main/webapp/file-download-popup-fragment.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml
index e1020c85e69..6fe3863b85f 100644
--- a/src/main/webapp/file-download-popup-fragment.xhtml
+++ b/src/main/webapp/file-download-popup-fragment.xhtml
@@ -138,7 +138,7 @@
                 value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}"
                 escape="false" />
         </div>
-        <p:fragment rendered="#{workingVersion.dataset.guestbook != null and workingVersion.dataset.guestbook.enabled and downloadPopupRequired}" id="guestbookUIFragment">
+        <p:fragment rendered="#{workingVersion.dataset.guestbook != null and workingVersion.dataset.guestbook.enabled and guestbookAndTermsPopupRequired}" id="guestbookUIFragment">
                 <!--
                 Only validate the active context. For example, the user might be filling out
                 the guestbook on the Preview tab rather than after clicking the "download file"

From 1da94e84c6846a84c0edf3cf10fcc7581dd2e9c7 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 22 Jun 2023 15:40:30 -0400
Subject: [PATCH 167/922] add missing actionListener

---
 src/main/webapp/file-download-button-fragment.xhtml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index cc566780769..2a8152d5395 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -128,6 +128,7 @@
                          action="#{guestbookResponseService.modifyDatafile(guestbookResponse, fileMetadata)}"
                          update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
                 <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
+                <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
@@ -167,7 +168,7 @@
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'original' )}"
                                update="@widgetVar(guestbookAndTermsPopup)"
                                oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                    <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                     <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                     <h:outputFormat value="#{bundle['file.downloadBtn.format.original']}">
                         <f:param value="#{fileMetadata.dataFile.originalFormatLabel}"/>
@@ -186,7 +187,7 @@
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                update="@widgetVar(guestbookAndTermsPopup)"
                                oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                    <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                     <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                     #{bundle['file.downloadBtn.format.tab']}
                 </p:commandLink>
@@ -205,7 +206,7 @@
                                    action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'RData' )}"
                                    update="@widgetVar(guestbookAndTermsPopup)"
                                    oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                                   <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                        <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                         <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                         #{bundle['file.downloadBtn.format.rdata']}
                     </p:commandLink>

From 58bc6c9a264ca7ef5caf44ad3fbc3a04a38f67fb Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 22 Jun 2023 15:40:48 -0400
Subject: [PATCH 168/922] missing actionListeners

---
 src/main/webapp/filesFragment.xhtml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 4c1a943b86e..7e1cb4ac4cd 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -444,6 +444,7 @@
                                    onclick="if (!testFilesSelected()) return false;" 
                                    action="#{DatasetPage.startDownloadSelectedOriginal()}"
                                    update="@form" oncomplete="showPopup();">
+                        <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                         <f:setPropertyActionListener target="#{DatasetPage.fileMetadataForAction}" value="#{null}"/>
                         <span class="glyphicon glyphicon-download-alt"/> #{bundle.download}
                     </p:commandLink>
@@ -462,6 +463,7 @@
                                            update="@form" oncomplete="showPopup();"
                                            onclick="if (!testFilesSelected()) return false;" 
                                            actionListener="#{DatasetPage.startDownloadSelectedOriginal()}">
+                                <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                                 <f:setPropertyActionListener target="#{DatasetPage.fileMetadataForAction}" value="#{null}"/>
                                 #{bundle.downloadOriginal}
                             </p:commandLink>

From 60c4db0d593b455a02d6df2ebe7801b8c9dbca5e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 22 Jun 2023 15:42:24 -0400
Subject: [PATCH 169/922] add missing params, change fileDownloadHelper to EJB

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 src/main/webapp/dataset.xhtml                           | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index f01fe00937e..40b0ba1a010 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -244,7 +244,7 @@ public enum DisplayMode {
     DatasetVersionUI datasetVersionUI;
     @Inject
     PermissionsWrapper permissionsWrapper;
-    @Inject
+    @EJB
     FileDownloadHelper fileDownloadHelper;
     @Inject
     ThumbnailServiceWrapper thumbnailServiceWrapper;
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index c39042a91ce..1cc8213ecd5 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1511,9 +1511,11 @@
                             <ui:param name="popupContext" value="downloadDataset"/>
                             <ui:param name="workingVersion" value="#{DatasetPage.workingVersion}"/>
                             <ui:param name="guestbookAndTermsPopupRequired" value="#{DatasetPage.guestbookAndTermsPopupRequired}"/>
+                            <ui:param name="guestbookPopupRequired" value="#{DatasetPage.guestbookPopupRequired}"/>
                             <ui:param name="fileMetadataForAction" value="#{DatasetPage.fileMetadataForAction}"/>
                             <ui:param name="hasRestrictedFile" value="#{DatasetPage.downloadingRestrictedFiles()}"/>
                             <ui:param name="guestbookResponse" value="#{DatasetPage.guestbookResponse}" rendered="#{!settingsWrapper.rsyncDownload}"/>
+                            <ui:param name="fileDownloadHelper" value="#{DatasetPage.fileDownloadHelper}"/>
                             <ui:param name="fileDownloadService" value="#{DatasetPage.fileDownloadService}"/>
                         </ui:include>
                     </p:dialog>

From 1e8495cff14955a12f7829a55fb7b2a434c537e6 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 22 Jun 2023 16:07:04 -0400
Subject: [PATCH 170/922] use Inject and restore getter/setter

---
 .../java/edu/harvard/iq/dataverse/DatasetPage.java     | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 40b0ba1a010..e890752c19c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -244,7 +244,7 @@ public enum DisplayMode {
     DatasetVersionUI datasetVersionUI;
     @Inject
     PermissionsWrapper permissionsWrapper;
-    @EJB
+    @Inject
     FileDownloadHelper fileDownloadHelper;
     @Inject
     ThumbnailServiceWrapper thumbnailServiceWrapper;
@@ -5476,6 +5476,14 @@ public FileDownloadServiceBean getFileDownloadService() {
     public void setFileDownloadService(FileDownloadServiceBean fileDownloadService) {
         this.fileDownloadService = fileDownloadService;
     }
+    
+    public FileDownloadHelper getFileDownloadHelper() {
+        return fileDownloadHelper;
+    }
+
+    public void setFileDownloadHelper(FileDownloadHelper fileDownloadHelper) {
+        this.fileDownloadHelper = fileDownloadHelper;
+    }
 
 
     public GuestbookResponseServiceBean getGuestbookResponseService() {

From 1400e07486223465641e289756c7d38577dbf00b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 23 Jun 2023 12:28:15 -0400
Subject: [PATCH 171/922] fix gb render conditions, initial fix for download
 buttons

---
 .../webapp/guestbook-terms-popup-fragment.xhtml    | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 829fa6539b9..ab75ffbe3e7 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -40,7 +40,7 @@
                     </div>
                 </div>
             </div>
-            <p:fragment rendered="#{workingVersion.dataset.guestbook != null and workingVersion.dataset.guestbook.enabled and guestbookAndTermsPopupRequired}" id="guestbookUIFragment">
+            <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((DatasetPage.termsGuestbookPopupAction != bundle.download) or DatasetPage.guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
                 <!--
                 Only validate the active context. For example, the user might be filling out
                 the guestbook on the Preview tab rather than after clicking the "download file"
@@ -147,11 +147,15 @@
             </p:fragment>
         </div>
         <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle.download}">
-            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" 
-                           action="#{fileDownloadHelper.requestAccessIndirect()}"    
-                           update="@([id$=requestAccessConsolidated]), @([id$=requestPanel]), @([id$=messagePanel]) @([id$=filesTable])" oncomplete="PF('requestAccessPopup').hide();">
+            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
+                             rendered="#{guestbookResponse.fileFormat != 'externalTool' and
+                                         guestbookResponse.fileFormat != 'package' and
+                                         popupContext != 'previewTab'}"
+                             actionListener="#{fileDownloadHelper.writeGuestbookAndStartDownload(guestbookResponse)}"
+                             update="guestbookUIFragment">
+                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
             </p:commandButton>
-            <button class="btn btn-link" onclick="PF('requestAccessPopup').hide();PF('blockDatasetForm').hide();" type="button">
+            <button class="btn btn-link" onclick="PF('guestbookAndTermsPopup').hide();PF('blockDatasetForm').hide();" type="button">
                 #{bundle.cancel}
             </button>
         </div>

From 647a5d8323bca1f3e6952bf0ae9a29a803ea97a0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 23 Jun 2023 12:28:28 -0400
Subject: [PATCH 172/922] comment typo

---
 src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index e44aeafcc4d..5279b10047a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -68,7 +68,7 @@ private boolean testResponseLength(String value) {
     // This helper method is called from the Download terms/guestbook/etc. popup,
     // when the user clicks the "ok" button. We use it, instead of calling
     // downloadServiceBean directly, in order to differentiate between single
-    // file downloads and multiple (batch) downloads - sice both use the same
+    // file downloads and multiple (batch) downloads - since both use the same
     // terms/etc. popup.
     public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) {
         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");

From 06b7cf157692dc0e12c3feb2d2104eb4968e02e0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 23 Jun 2023 12:28:57 -0400
Subject: [PATCH 173/922] avoid duplicate file counting and extra popup

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 26 +++++--------------
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index e890752c19c..3693fe11c18 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3172,17 +3172,12 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
             return false;
         }
 
-        for (FileMetadata fmd : this.selectedFiles) {
-            if (this.fileDownloadHelper.canDownloadFile(fmd)) {
-                getSelectedDownloadableFiles().add(fmd);
-                DataFile dataFile = fmd.getDataFile();
-                if (downloadOriginal && dataFile.isTabularData()) {
-                    bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize();
-                } else {
-                    bytes += dataFile.getFilesize();
-                }
+        for (FileMetadata fmd : getSelectedDownloadableFiles()) {
+            DataFile dataFile = fmd.getDataFile();
+            if (downloadOriginal && dataFile.isTabularData()) {
+                bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize();
             } else {
-                getSelectedNonDownloadableFiles().add(fmd);
+                bytes += dataFile.getFilesize();
             }
         }
 
@@ -3193,13 +3188,6 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
             return false;
         }
 
-        
-//QDRADA handle new state from
-        if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()){
-            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
-        }
-        
-        
         // If some of the files were restricted and we had to drop them off the
         // list, and NONE of the files are left on the downloadable list
         // - we show them a "you're out of luck" popup:
@@ -3213,11 +3201,9 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
             return true;
         }
 
-      //QDRADA - still needed?
-/*        if (guestbookRequired) {
+        if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()) {
             setValidateFilesOutcome("GuestbookRequired");
         }
-*/
         return true;
 
     }

From 2b3a1cc8925c154b43cb1b91721df8c87287ee70 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 23 Jun 2023 15:30:51 -0400
Subject: [PATCH 174/922] switch to guestbook-terms-popup-fragment

---
 src/main/webapp/file-download-popup-fragment.xhtml | 1 +
 src/main/webapp/file.xhtml                         | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml
index 6fe3863b85f..65b5cc9a851 100644
--- a/src/main/webapp/file-download-popup-fragment.xhtml
+++ b/src/main/webapp/file-download-popup-fragment.xhtml
@@ -1,3 +1,4 @@
+<!--  TODO - remove! -->
 <ui:composition xmlns="http://www.w3.org/1999/xhtml"
                 xmlns:h="http://java.sun.com/jsf/html"
                 xmlns:f="http://java.sun.com/jsf/core"
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index 2c259d708b8..05bd345c011 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -362,7 +362,7 @@
                                    rendered="#{FilePage.toolsWithPreviews.size() > 0 and fileDownloadHelper.canDownloadFile(FilePage.fileMetadata)}">
                                 <!-- PREVIEW TERMS/GUESTBOOK FORM -->
                                 <ui:fragment rendered="#{FilePage.guestbookAndTermsPopupRequired and !FilePage.termsMet}">
-                                    <ui:include src="file-download-popup-fragment.xhtml">
+                                    <ui:include src="guestbook-terms-popup-fragment.xhtml">
                                         <ui:param name="popupContext" value="previewTab"/>
                                         <ui:param name="workingVersion" value="#{FilePage.fileMetadata.datasetVersion}"/>
                                         <ui:param name="guestbookAndTermsPopupRequired" value="#{FilePage.guestbookAndTermsPopupRequired}"/>

From 51005bad3afd714aa8edbbbb03eb0faf604470c2 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 23 Jun 2023 15:31:22 -0400
Subject: [PATCH 175/922] update fragment with license info and buttons for
 other views

---
 .../guestbook-terms-popup-fragment.xhtml      | 195 +++++++++++++++---
 1 file changed, 168 insertions(+), 27 deletions(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index ab75ffbe3e7..4bddbb0dd38 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -9,37 +9,138 @@
                 xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs">
 
         <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
-        <p class="help-block">
-            #{someActivelyEmbargoedFiles ? bundle['file.requestAccessTermsDialog.embargoed.tip'] : bundle['file.requestAccessTermsDialog.tip']}
-        </p>
-        <p class="help-block" jsf:rendered="#{someActivelyEmbargoedFiles}">
-            #{bundle['file.requestAccessTermsDialog.embargoed']}
-        </p>
-        <div class="form-horizontal">
-            <div class="form-group" jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.license and workingVersion.termsOfUseAndAccess.license.name != 'CC0' and !empty workingVersion.termsOfUseAndAccess.termsOfUse}">
-                <label class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.termsOfUse']}
+        <p:focus context="guestbookUIFragment"/>
+        <ui:fragment rendered="#{DatasetPage.termsGuestbookPopupAction != bundle.download}">
+            <p class="help-block">
+                #{someActivelyEmbargoedFiles ? bundle['file.requestAccessTermsDialog.embargoed.tip'] : bundle['file.requestAccessTermsDialog.tip']}
+            </p>
+            <p class="help-block" jsf:rendered="#{someActivelyEmbargoedFiles}">
+                #{bundle['file.requestAccessTermsDialog.embargoed']}
+            </p>
+        </ui:fragment>
+        <ui:fragment rendered="#{DatasetPage.termsGuestbookPopupAction != bundle.download}">
+            <p class="help-block">
+                #{bundle['file.downloadDialog.tip']}
+            </p>
+        </ui:fragment>
+        <div class="form-horizontal terms">
+        <div class="form-group">
+            <label jsf:for="fdLicense" class="col-sm-3 control-label">
+                #{bundle['file.dataFilesTab.terms.list.license']} </label>
+            <div class="col-sm-9">
+                <p class="help-block">
+                    <h:outputText
+                        value="#{bundle['file.dataFilesTab.terms.list.license.view.description']}"
+                        escape="false" />
+                </p>
+                <p>
+                    <o:importFunctions
+                        type="edu.harvard.iq.dataverse.dataset.DatasetUtil" />
+                    <img
+                        src="#{DatasetUtil:getLicenseIcon(workingVersion)}"
+                        title="#{DatasetUtil:getLicenseDescription(workingVersion)}"
+                        style="display: none" onload="this.style.display='inline'" />
+                    <a 
+                        href="#{DatasetUtil:getLicenseURI(workingVersion)}"
+                        title="#{DatasetUtil:getLicenseDescription(workingVersion)}"
+                        target="_blank">#{DatasetUtil:getLicenseName(workingVersion)}
+                    </a>
+                    <ui:fragment
+                        rendered="#{empty workingVersion.termsOfUseAndAccess.license}">
+                        <h:outputText
+                            value="#{bundle['file.dataFilesTab.terms.list.license.customterms.txt']}"
+                            escape="false" />
+                    </ui:fragment>
+                </p>
+            </div>
+        </div>
+
+        <ui:fragment rendered="#{empty workingVersion.termsOfUseAndAccess.license}">
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.termsOfUse}">
+                <label jsf:for="fdTermsOfuse" class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.termsOfUse']} 
                 </label>
-                <div class="col-sm-6">
-                    <div class="panel panel-default">
-                        <div class="panel-body read-terms">
-                            <h:outputText value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfUse)}" escape="false" />
-                        </div>
-                    </div>
-                </div>
+                <h:outputText id="fdTermsOfUse" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfUse)}"
+                    escape="false" />
             </div>
-            <div class="form-group" jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.termsOfAccess}">
-                <label class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess']}
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.confidentialityDeclaration}">
+                <label jsf:for="fdConfidentialityDeclaration"
+                    class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.declaration']}
                 </label>
-                <div class="col-sm-6">
-                    <div class="panel panel-default">
-                        <div class="panel-body read-terms">
-                            <h:outputText value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}" escape="false" />
-                        </div>
-                    </div>
-                </div>
+                <h:outputText id="fdConfidentialityDeclaration" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.confidentialityDeclaration)}"
+                    escape="false" />
+            </div>
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.specialPermissions}">
+                <label jsf:for="fdSpecialPermissions" class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.permissions']}
+                </label>
+                <h:outputText id="fdSpecialPermissions" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.specialPermissions)}"
+                    escape="false" />
+            </div>
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.restrictions}">
+                <label jsf:for="fdRestrictions" class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.restrictions']}
+                </label>
+                <h:outputText id="fdRestrictions" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.restrictions)}"
+                    escape="false" />
+            </div>
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.citationRequirements}">
+                <label jsf:for="fdCitationRequirements"
+                    class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.citationRequirements']}
+                </label>
+                <h:outputText id="fdCitationRequirements" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.citationRequirements)}"
+                    escape="false" />
+            </div>
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.depositorRequirements}">
+                <label jsf:for="fdDepositorRequirements"
+                    class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.depositorRequirements']}
+                </label>
+                <h:outputText id="fdDepositorRequirements" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.depositorRequirements)}"
+                    escape="false" />
             </div>
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.conditions}">
+                <label jsf:for="fdConditions" class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.conditions']}
+                </label>
+                <h:outputText id="fdConditions" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.conditions)}"
+                    escape="false" />
+            </div>
+            <div class="form-group"
+                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.disclaimer}">
+                <label jsf:for="fdDisclaimer" class="col-sm-3 control-label">
+                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer']}
+                </label>
+                <h:outputText id="fdDisclaimer" class="col-sm-6"
+                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.disclaimer)}"
+                    escape="false" />
+            </div>
+        </ui:fragment>
+        <div class="form-group"
+            jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.termsOfAccess and hasRestrictedFile}">
+            <label jsf:for="fdTermsOfAccess" class="col-sm-3 control-label">
+                #{bundle['file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess']}
+            </label>
+            <h:outputText id="fdTermsOfAccess" class="col-sm-6"
+                value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}"
+                escape="false" />
+        </div>
             <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((DatasetPage.termsGuestbookPopupAction != bundle.download) or DatasetPage.guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
                 <!--
                 Only validate the active context. For example, the user might be filling out
@@ -147,6 +248,25 @@
             </p:fragment>
         </div>
         <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle.download}">
+        <!--
+            The "process" directive below is very important. Without it, the
+            setters on the GuestbookResponse object can be called twice leading
+            to form values (name, email, etc) to be overwritten by the object in
+            the other context. For example, "name" in the Preview tab could get
+            overwritten by "name" in the download popup with either a blank
+            value or a prefilled value, leading to a botched guestbook entry.
+            
+            Experimentation was done with adding process="downloadPopup" to the
+            non-Preview tab buttons but this caused a ComponentNotFoundException
+            and didn't solve any problems. If you add logging to print out
+            setName from Guestbook response, you can see that the setters are
+            called on the GuestbookResponse object from the Preview tab but they
+            are later overwritten by the setters on the GuestbookResponse object
+            from the Download popup.
+            -->
+            <!--REGULAR DOWNLOAD BUTTON, NO EXTERNAL TOOL, NOT THE PREVIEW TAB-->
+            <!--Note: the guestbookResponse.fileFormat is being set in xhtml via the initial download buttons in file-download-button-fragment.xhtml -->
+            
             <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
                              rendered="#{guestbookResponse.fileFormat != 'externalTool' and
                                          guestbookResponse.fileFormat != 'package' and
@@ -154,6 +274,27 @@
                              actionListener="#{fileDownloadHelper.writeGuestbookAndStartDownload(guestbookResponse)}"
                              update="guestbookUIFragment">
                 <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
+            </p:commandButton>
+                        <!--PREVIEW TAB BUTTON-->
+            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
+                             rendered="#{popupContext == 'previewTab'}"
+                             actionListener="#{FilePage.showPreview(guestbookResponse)}"
+                             process="previewTab"
+                             update="fileForm:tabView">
+                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
+            </p:commandButton>
+            <!--EXTERNAL TOOL BUTTON-->
+            <!--On the dataset page (but not the file page), "tool" is null so we get the tool from the guestbookResponse.-->
+            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" rendered="#{guestbookResponse.fileFormat == 'externalTool'}"
+                             action="#{fileDownloadHelper.writeGuestbookAndLaunchExploreTool(guestbookResponse, fileMetadata, tool)}"
+                             update="guestbookUIFragment">
+                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
+            </p:commandButton>
+            <!--PACKAGE FILE BUTTON-->
+            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" rendered="#{guestbookResponse.fileFormat == 'package'}"
+                             actionListener="#{fileDownloadHelper.writeGuestbookAndLaunchPackagePopup(guestbookResponse)}"
+                             update="guestbookUIFragment">
+                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
             </p:commandButton>
             <button class="btn btn-link" onclick="PF('guestbookAndTermsPopup').hide();PF('blockDatasetForm').hide();" type="button">
                 #{bundle.cancel}

From 7e1036e4cc399c3be5204e5fb72f998c5103f998 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 23 Jun 2023 15:31:49 -0400
Subject: [PATCH 176/922] update grant access to set request state, add todos

---
 .../dataverse/ManageFilePermissionsPage.java  | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index 36a4cff6882..682334a98ac 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -72,6 +72,8 @@ public class ManageFilePermissionsPage implements java.io.Serializable {
     DataverseRequestServiceBean dvRequestService;
     @Inject
     PermissionsWrapper permissionsWrapper;
+    @EJB
+    FileAccessRequestServiceBean fileAccessRequestService;
     
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     EntityManager em;
@@ -408,15 +410,24 @@ public void grantAccess(ActionEvent evt) {
                         sendNotification = true;
                     }
                     // remove request, if it exist
-                    if (file.removeFileAccessRequester(roleAssignee)) {
-                        datafileService.save(file);
+                    for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
+                            if (file.getFileAccessRequesters().remove(au)) {
+                                List<FileAccessRequest> fileAccessRequests = fileAccessRequestService.findAllByAuthenticatedUserIdAndRequestState(au.getId(), FileAccessRequest.RequestState.CREATED);
+                                for(FileAccessRequest far : fileAccessRequests){
+                                    far.setStateGranted();
+                                    fileAccessRequestService.save(far);
+                                }
+                                file.setFileAccessRequests(fileAccessRequests); 
+                                datafileService.save(file);
+                            }       
                     }
                 }
+
             }
 
             if (sendNotification) {
                 for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
-                    userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.GRANTFILEACCESS, dataset.getId());
+                    userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.GRANTFILEACCESS, dataset.getId());                
                 }
              }
         }
@@ -444,7 +455,14 @@ private void grantAccessToRequests(AuthenticatedUser au, List<DataFile> files) {
         DataverseRole fileDownloaderRole = roleService.findBuiltinRoleByAlias(DataverseRole.FILE_DOWNLOADER);
         for (DataFile file : files) {
             if (assignRole(au, file, fileDownloaderRole)) {
+                //TODO - why remove requests just to set them again?
                 if (file.removeFileAccessRequester(au)) {
+                    List<FileAccessRequest> fileAccessRequests = fileAccessRequestService.findAll(au.getId(), file.getId(), FileAccessRequest.RequestState.CREATED);
+                    for(FileAccessRequest far : fileAccessRequests){
+                        far.setStateGranted();
+                        fileAccessRequestService.save(far);
+                    }
+                    file.setFileAccessRequests(fileAccessRequests); 
                     datafileService.save(file);
                 }
                 actionPerformed = true;
@@ -476,9 +494,12 @@ public void rejectAccessToAllRequests(AuthenticatedUser au) {
     private void rejectAccessToRequests(AuthenticatedUser au, List<DataFile> files) {
         boolean actionPerformed = false;
         for (DataFile file : files) {
-            file.removeFileAccessRequester(au);
-            datafileService.save(file);
-            actionPerformed = true;
+            if(file.removeFileAccessRequester(au)) {
+            
+                // TODO - set FileAccessRequest.RequestState to REJECTED
+                datafileService.save(file);
+                actionPerformed = true;
+            }
         }
 
         if (actionPerformed) {

From ce1ad60505d7bc4d799043a58637aa634638e22b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Sat, 24 Jun 2023 09:33:00 -0400
Subject: [PATCH 177/922] only show newly CREATED requests for now

need to revisit the lifecycle of requests and specify which states allow
granting, how states are changed, etc.
---
 .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index 682334a98ac..6bf548e65d3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -5,6 +5,7 @@
  */
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.FileAccessRequest.RequestState;
 import edu.harvard.iq.dataverse.api.Util;
 import edu.harvard.iq.dataverse.authorization.AuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
@@ -187,7 +188,9 @@ private void initMaps() {
                         AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getRequester().getUserIdentifier());
                         fileAccessRequestMap.put(withProvider, requestedFiles);
                     }
-                    requestedFiles.add(fileAccessRequest);
+                    if(fileAccessRequest.getState() == RequestState.CREATED){
+                        requestedFiles.add(fileAccessRequest);
+                    }
                 }
             }
         }

From a7b428862b42d0b69a0054fcb00ec40bcc6b51f8 Mon Sep 17 00:00:00 2001
From: Kim Jin Yeon <hourglass458@gmail.com>
Date: Mon, 3 Jul 2023 12:43:57 +0900
Subject: [PATCH 178/922] Add description on Basic File Aceess about
 FilePIDsEnabled

---
 doc/sphinx-guides/source/api/dataaccess.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index e76ea167587..98df2d5a046 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -83,7 +83,7 @@ Basic access URI:
 
 ``/api/access/datafile/$id``
 
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``.
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. (FilePIDsEnabled 옵션� 켜져 있� 때만 �작하는 것 언급하고 FilePIDsEnabled 설명 문서 �� 첨부하기)
 
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 

From e3fd94502d4bc60541cd233847e92367d4be054d Mon Sep 17 00:00:00 2001
From: Minji Woo <minjiwoo03@gmail.com>
Date: Mon, 3 Jul 2023 13:05:52 +0900
Subject: [PATCH 179/922] Update dataaccess.rst file access

General description of the basic file access: FilePIDsEnabled
---
 doc/sphinx-guides/source/api/dataaccess.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 98df2d5a046..2ecdc0d2a23 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -83,7 +83,7 @@ Basic access URI:
 
 ``/api/access/datafile/$id``
 
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. (FilePIDsEnabled 옵션� 켜져 있� 때만 �작하는 것 언급하고 FilePIDsEnabled 설명 문서 �� 첨부하기)
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin.  
 
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 

From 89353d4fb8135b5f957cea799bc5347c6ae9c556 Mon Sep 17 00:00:00 2001
From: kmina02 <79454352+kmina02@users.noreply.github.com>
Date: Mon, 3 Jul 2023 13:12:14 +0900
Subject: [PATCH 180/922] Update dataaccess.rst

put further link about FilePIDsEnabled
---
 doc/sphinx-guides/source/api/dataaccess.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 2ecdc0d2a23..15a93ab436a 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -84,7 +84,7 @@ Basic access URI:
 ``/api/access/datafile/$id``
 
 .. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin.  
-
+For further information, refer to https://guides.dataverse.org/en/5.13/installation/config.html?highlight=filepidsenabled#id255
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 
     GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB

From ca796d785b3470805c98adeb6ed86e843c5059b4 Mon Sep 17 00:00:00 2001
From: Kim Jin Yeon <hourglass458@gmail.com>
Date: Mon, 3 Jul 2023 13:22:48 +0900
Subject: [PATCH 181/922] Update dataaccess.rst

---
 doc/sphinx-guides/source/api/dataaccess.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 15a93ab436a..e4cde143d29 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -83,8 +83,7 @@ Basic access URI:
 
 ``/api/access/datafile/$id``
 
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin.  
-For further information, refer to https://guides.dataverse.org/en/5.13/installation/config.html?highlight=filepidsenabled#id255
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to https://guides.dataverse.org/en/latest/installation/config.html?highlight=pidsenabled
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 
     GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB

From 20e76599b78b5d77dee89784e99241b4095850b2 Mon Sep 17 00:00:00 2001
From: Kim Jin Yeon <hourglass458@gmail.com>
Date: Mon, 3 Jul 2023 14:06:03 +0900
Subject: [PATCH 182/922] Update dataaccess.rst

---
 doc/sphinx-guides/source/api/dataaccess.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index e4cde143d29..99e7f7eca92 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -84,6 +84,7 @@ Basic access URI:
 ``/api/access/datafile/$id``
 
 .. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to https://guides.dataverse.org/en/latest/installation/config.html?highlight=pidsenabled
+
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 
     GET http://$SERVER/api/access/datafile/:persistentId?persistentId=doi:10.5072/FK2/J8SJZB

From 17290efa53ac7b692a1d851b0d4e10a7d4cd8aea Mon Sep 17 00:00:00 2001
From: Kim Jin Yeon <hourglass458@gmail.com>
Date: Mon, 3 Jul 2023 14:12:33 +0900
Subject: [PATCH 183/922] Update dataaccess.rst

---
 doc/sphinx-guides/source/api/dataaccess.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 99e7f7eca92..7b1feed7814 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -83,7 +83,7 @@ Basic access URI:
 
 ``/api/access/datafile/$id``
 
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to https://guides.dataverse.org/en/latest/installation/config.html?highlight=pidsenabled
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to `FilePIDsEnabled <https://guides.dataverse.org/en/latest/installation/config.html?highlight=pidsenabled>`_ 
 
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 

From 73c3235c2442e285274ed7857098eaebdd0297d6 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 4 Jul 2023 10:39:31 +0100
Subject: [PATCH 184/922] Stash: getVersionFiles API extension with pagination
 and order criteria WIP

---
 .../dataverse/DatasetVersionServiceBean.java  | 76 ++++++++++++++++++-
 .../harvard/iq/dataverse/api/Datasets.java    | 12 ++-
 2 files changed, 83 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 9f272ec6877..5100f9d26eb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -49,7 +49,21 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
     private static final Logger logger = Logger.getLogger(DatasetVersionServiceBean.class.getCanonicalName());
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
-    
+
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm WHERE fm.datasetVersion.id=:datasetVersionId ORDER BY fm.label";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = dvo.id"
+            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.filesize";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.contentType";
+
     @EJB
     DatasetServiceBean datasetService;
     
@@ -150,7 +164,19 @@ public DatasetVersion getDatasetVersion(){
             return this.datasetVersionForResponse;
         }                
     } // end RetrieveDatasetVersionResponse
-    
+
+    /**
+     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas(DatasetVersion, Integer, Integer, FileMetadatasOrderCriteria)}
+     */
+    public enum FileMetadatasOrderCriteria {
+        NameAZ,
+        NameZA,
+        Newest,
+        Oldest,
+        Size,
+        Type
+    }
+
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -1210,4 +1236,50 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
+
+    /**
+     * Returns a FileMetadata list of files in the specified DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param limit for pagination, can be null
+     * @param offset for pagination, can be null
+     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
+     * @return a FileMetadata list of the specified DatasetVersion
+     */
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
+        Query query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria))
+                .setParameter("datasetVersionId", datasetVersion.getId());
+        if (limit != null) {
+            query.setMaxResults(limit);
+        }
+        if (offset != null) {
+            query.setFirstResult(offset);
+        }
+        return query.getResultList();
+    }
+
+    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
+        String queryString;
+        switch (orderCriteria) {
+            case NameZA:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
+                break;
+            case Newest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
+                break;
+            case Oldest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
+                break;
+            case Size:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
+                break;
+            case Type:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
+                break;
+            default:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
+                break;
+        }
+        return queryString;
+    }
 } // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 8c1390b597e..5aef4302631 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -493,9 +493,15 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
-    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return response( req -> ok( jsonFileMetadatas(
-                         getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getFileMetadatas())), getRequestUser(crc));
+    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response( req -> {
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            try {
+                return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ)));
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
+            }
+        }, getRequestUser(crc));
     }
     
     @GET

From dc31788c09eb168565ea84c04e4cf9ed9ccd1387 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 4 Jul 2023 11:43:58 +0100
Subject: [PATCH 185/922] Stash: IT tests for getVersionFiles API endpoint WIP

---
 .../dataverse/DatasetVersionServiceBean.java  |  2 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 32 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 31 +++++++++++++++---
 3 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 5100f9d26eb..45edfb01777 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -166,7 +166,7 @@ public DatasetVersion getDatasetVersion(){
     } // end RetrieveDatasetVersionResponse
 
     /**
-     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas(DatasetVersion, Integer, Integer, FileMetadatasOrderCriteria)}
+     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
      */
     public enum FileMetadatasOrderCriteria {
         NameAZ,
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 687ab453d24..973818776f3 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -9,6 +9,7 @@
 
 import java.util.logging.Logger;
 
+import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.skyscreamer.jsonassert.JSONAssert;
@@ -3202,4 +3203,35 @@ public void getDatasetVersionCitation() {
                 // We check that the returned message contains information expected for the citation string
                 .body("data.message", containsString("DRAFT VERSION"));
     }
+
+    @Test
+    public void getVersionFiles() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        String testFileName1 = "test_1.txt";
+        String testFileName2 = "test_2.txt";
+        String testFileName3 = "test_3.txt";
+        String testFileName4 = "test_4.txt";
+
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[50], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[100], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[200], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName4, new byte[300], apiToken);
+
+        Response getVersionFilesResponseOrderNameAZ = UtilIT.getVersionFiles(datasetId, ":latest", 2, 0, DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ.toString(), apiToken);
+        getVersionFilesResponseOrderNameAZ.prettyPrint();
+
+        // TODO
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 6e24d0a0ecb..e0f648e48aa 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -4,6 +4,8 @@
 import com.jayway.restassured.http.ContentType;
 import com.jayway.restassured.path.json.JsonPath;
 import com.jayway.restassured.response.Response;
+
+import java.io.*;
 import java.util.UUID;
 import java.util.logging.Logger;
 import javax.json.Json;
@@ -12,8 +14,6 @@
 import javax.json.JsonObject;
 import javax.ws.rs.client.Client;
 import javax.ws.rs.client.ClientBuilder;
-import java.io.File;
-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -30,7 +30,6 @@
 import com.mashape.unirest.http.Unirest;
 import com.mashape.unirest.http.exceptions.UnirestException;
 import com.mashape.unirest.request.GetRequest;
-import java.io.InputStream;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import java.util.Base64;
 import org.apache.commons.io.IOUtils;
@@ -49,8 +48,11 @@
 import edu.harvard.iq.dataverse.DatasetFieldType;
 import edu.harvard.iq.dataverse.DatasetFieldValue;
 import edu.harvard.iq.dataverse.util.StringUtil;
-import java.io.StringReader;
+
 import java.util.Collections;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -3240,4 +3242,25 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/citation");
         return response;
     }
+
+    static Response getVersionFiles(Integer datasetId, String version, int limit, int offset, String orderCriteria, String apiToken) {
+        Response response = given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .contentType("application/json")
+                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files?limit=" + limit + "&offset=" + offset + "&orderCriteria=" + orderCriteria);
+        return response;
+    }
+
+    static Response createAndUploadTestFile(String persistentId, String testFileName, byte[] testFileContentInBytes, String apiToken) throws IOException {
+        Path pathToTempDir = Paths.get(Files.createTempDirectory(null).toString());
+        String pathToTestFile = pathToTempDir + File.separator + testFileName;
+        File testFile = new File(pathToTestFile);
+        FileOutputStream fileOutputStream = new FileOutputStream(testFile);
+
+        fileOutputStream.write(testFileContentInBytes);
+        fileOutputStream.flush();
+        fileOutputStream.close();
+
+        return uploadZipFileViaSword(persistentId, pathToTestFile, apiToken);
+    }
 }

From fff211910e4a060f873e8949e3ea2d9653ea46d5 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 4 Jul 2023 21:44:45 +0100
Subject: [PATCH 186/922] Added: IT tests for getVersionFiles API endpoint

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 102 ++++++++++++++++--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  18 +++-
 2 files changed, 108 insertions(+), 12 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 973818776f3..e994536f03a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -94,7 +94,6 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.junit.Assert.assertFalse;
 
 
 public class DatasetsIT {
@@ -3223,15 +3222,104 @@ public void getVersionFiles() throws IOException {
         String testFileName2 = "test_2.txt";
         String testFileName3 = "test_3.txt";
         String testFileName4 = "test_4.txt";
+        String testFileName5 = "test_5.png";
 
         UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[50], apiToken);
-        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[100], apiToken);
-        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[200], apiToken);
-        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName4, new byte[300], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName5, new byte[300], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName4, new byte[400], apiToken);
 
-        Response getVersionFilesResponseOrderNameAZ = UtilIT.getVersionFiles(datasetId, ":latest", 2, 0, DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ.toString(), apiToken);
-        getVersionFilesResponseOrderNameAZ.prettyPrint();
+        String testDatasetVersion = ":latest";
 
-        // TODO
+        // Test pagination and NameAZ order criteria (the default criteria)
+        int testPageSize = 2;
+
+        // Test page 1
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, apiToken);
+
+        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(testPageSize, fileMetadatasCount);
+
+        getVersionFilesResponsePaginated.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1))
+                .body("data[1].label", equalTo(testFileName2));
+
+        // Test page 2
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, apiToken);
+
+        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(testPageSize, fileMetadatasCount);
+
+        getVersionFilesResponsePaginated.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName3))
+                .body("data[1].label", equalTo(testFileName4));
+
+        // Test page 3 (last)
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, apiToken);
+
+        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        getVersionFilesResponsePaginated.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName5));
+
+        // Test NameZA order criteria
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+
+        getVersionFilesResponseNameZACriteria.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName5))
+                .body("data[1].label", equalTo(testFileName4))
+                .body("data[2].label", equalTo(testFileName3))
+                .body("data[3].label", equalTo(testFileName2))
+                .body("data[4].label", equalTo(testFileName1));
+
+        // Test Newest order criteria
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+
+        getVersionFilesResponseNewestCriteria.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName4))
+                .body("data[1].label", equalTo(testFileName5))
+                .body("data[2].label", equalTo(testFileName3))
+                .body("data[3].label", equalTo(testFileName2))
+                .body("data[4].label", equalTo(testFileName1));
+
+        // Test Oldest order criteria
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+
+        getVersionFilesResponseOldestCriteria.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1))
+                .body("data[1].label", equalTo(testFileName2))
+                .body("data[2].label", equalTo(testFileName3))
+                .body("data[3].label", equalTo(testFileName5))
+                .body("data[4].label", equalTo(testFileName4));
+
+        // Test Size order criteria
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+
+        getVersionFilesResponseSizeCriteria.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1))
+                .body("data[1].label", equalTo(testFileName3))
+                .body("data[2].label", equalTo(testFileName2))
+                .body("data[3].label", equalTo(testFileName5))
+                .body("data[4].label", equalTo(testFileName4));
+
+        // Test Type order criteria
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+
+        getVersionFilesResponseTypeCriteria.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName5))
+                .body("data[1].label", equalTo(testFileName1))
+                .body("data[2].label", equalTo(testFileName2))
+                .body("data[3].label", equalTo(testFileName3))
+                .body("data[4].label", equalTo(testFileName4));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e0f648e48aa..37e287ab19c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3243,12 +3243,20 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, int limit, int offset, String orderCriteria, String apiToken) {
-        Response response = given()
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String orderCriteria, String apiToken) {
+        RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .contentType("application/json")
-                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files?limit=" + limit + "&offset=" + offset + "&orderCriteria=" + orderCriteria);
-        return response;
+                .contentType("application/json");
+        if (limit != null) {
+            requestSpecification = requestSpecification.queryParam("limit", limit);
+        }
+        if (offset != null) {
+            requestSpecification = requestSpecification.queryParam("offset", offset);
+        }
+        if (orderCriteria != null) {
+            requestSpecification = requestSpecification.queryParam("orderCriteria", orderCriteria);
+        }
+        return requestSpecification.get("/api/datasets/" + datasetId + "/versions/" + version + "/files");
     }
 
     static Response createAndUploadTestFile(String persistentId, String testFileName, byte[] testFileContentInBytes, String apiToken) throws IOException {

From e8951a48b5e252a00f5f40a6bda14a8642d89dbe Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 5 Jul 2023 09:58:38 +0100
Subject: [PATCH 187/922] Removed: unused imports

---
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 37e287ab19c..591ab1c4222 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -50,8 +50,6 @@
 import edu.harvard.iq.dataverse.util.StringUtil;
 
 import java.util.Collections;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;

From 6264fa43a34e3946e2933716ad6e375350022186 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 5 Jul 2023 19:12:45 +0100
Subject: [PATCH 188/922] Added: publication date field to data file payload

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 601d1c34e17..ed06a2c360b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -687,6 +687,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata) {
                 .add("fileMetadataId", fileMetadata.getId())
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate",  df.getCreateDateFormattedYYYYMMDD())
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
                 .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables()))
                 .add("varGroups", fileMetadata.getVarGroups().isEmpty()
                         ? JsonPrinter.jsonVarGroup(fileMetadata.getVarGroups())

From 6687e2875c8b723df64c91404fde66de64a8e5cf Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 5 Jul 2023 14:38:38 -0400
Subject: [PATCH 189/922] call correct script

---
 src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index 5279b10047a..b5f2ee97094 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -290,7 +290,7 @@ public void handleCommandLinkClick(FileMetadata fmd){
         
         if (FileUtil.isRequestAccessPopupRequired(fmd.getDatasetVersion())){
             addFileForRequestAccess(fmd.getDataFile());
-            PrimeFaces.current().executeScript("PF('requestAccessPopup').show()");
+            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show()");
         } else {
             requestAccess(fmd.getDataFile());
         }

From 99f09fa188b9e3763d91c7a2398f2ae9ad22d2e0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 5 Jul 2023 14:39:11 -0400
Subject: [PATCH 190/922] update method call to match changes in
 FileDownloadServiceBean

---
 .../iq/dataverse/engine/command/impl/RequestAccessCommand.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index db0054e375e..a8c6150b21e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -88,9 +88,8 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         file.addFileAccessRequester(requester);
         requester.getFileAccessRequests().add(fileAccessRequest);
         if (sendNotification) {
-            //QDRADA
             logger.info("ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);"); 
-            //ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);
+            ctxt.fileDownload().sendRequestFileAccessNotification(this.file.getOwner(), this.file.getId(), requester);
         }
         return ctxt.files().save(file);
     }

From ceeb13fb95bdbe2d23c4a48a97612b33d7927353 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 5 Jul 2023 17:27:01 -0400
Subject: [PATCH 191/922] update flyway numbering

---
 ...estbook-on-request.sql => V5.13.0.3__guestbook-on-request.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V5.13.0.2__guestbook-on-request.sql => V5.13.0.3__guestbook-on-request.sql} (100%)

diff --git a/src/main/resources/db/migration/V5.13.0.2__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
similarity index 100%
rename from src/main/resources/db/migration/V5.13.0.2__guestbook-on-request.sql
rename to src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql

From 639cff8287f3fcfc5179d2b75a888efe78f07216 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 6 Jul 2023 08:01:52 +0100
Subject: [PATCH 192/922] Added: getCountGuestbookResponsesByDataFileId API
 endpoint

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 15 +++++++-
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 36 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 ++++
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index f6eda085c95..467341f4077 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -12,6 +12,7 @@
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
 import edu.harvard.iq.dataverse.FileMetadata;
+import edu.harvard.iq.dataverse.GuestbookResponseServiceBean;
 import edu.harvard.iq.dataverse.TermsOfUseAndAccessValidator;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
@@ -102,7 +103,9 @@ public class Files extends AbstractApiBean {
     SettingsServiceBean settingsService;
     @Inject
     MakeDataCountLoggingServiceBean mdcLogService;
-    
+    @Inject
+    GuestbookResponseServiceBean guestbookResponseService;
+
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
     
@@ -818,4 +821,14 @@ public Response getExternalToolFMParams(@Context ContainerRequestContext crc, @P
     public Response getFixityAlgorithm() {
         return ok(systemConfig.getFileFixityChecksumAlgorithm().toString());
     }
+
+    @GET
+    @Path("{id}/guestbookResponses/count")
+    public Response getCountGuestbookResponsesByDataFileId(@PathParam("id") String dataFileId) {
+        try {
+            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(Long.parseLong(dataFileId)).toString());
+        } catch (NumberFormatException nfe) {
+            return badRequest("File identifier has to be numeric.");
+        }
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index ed4d255ab74..49b41d1e0e0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2018,4 +2018,40 @@ public void testDeleteFile() {
                 .body("data.files[0]", equalTo(null))
                 .statusCode(OK.getStatusCode());
     }
+
+    @Test
+    public void testGetCountGuestbookResponsesByDataFileId() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Publish collection and dataset
+        UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+        UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        // Download test file
+        Integer testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+
+        Response downloadResponse = UtilIT.downloadFile(testFileId, apiToken);
+        downloadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get count guestbook responses and assert it is 1
+        Response getGuestbookResponsesByDataFileIdResponse = UtilIT.getCountGuestbookResponsesByDataFileId(testFileId, apiToken);
+        getGuestbookResponsesByDataFileIdResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.message", equalTo("1"));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 591ab1c4222..6c2543bcf82 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3269,4 +3269,10 @@ static Response createAndUploadTestFile(String persistentId, String testFileName
 
         return uploadZipFileViaSword(persistentId, pathToTestFile, apiToken);
     }
+
+    static Response getCountGuestbookResponsesByDataFileId(Integer dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/files/" + dataFileId + "/guestbookResponses/count");
+    }
 }

From 886a5082b2af8f9435b4bd9d0d1ac0a6e532c0a1 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 6 Jul 2023 12:27:40 +0100
Subject: [PATCH 193/922] Added: canDownloadFile method to
 FileDownloadServiceBean

---
 .../iq/dataverse/FileDownloadServiceBean.java |  46 +++-
 .../FileDownloadServiceBeanTest.java          | 246 ++++++++++++++++++
 2 files changed, 291 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index a90489be29a..5c490b18ecc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -572,5 +572,49 @@ public String getDirectStorageLocatrion(String storageLocation) {
             
         return null; 
     }
-    
+
+    /**
+     *
+     * Checks if a user can download a file based on the file metadata and the permissions of the user
+     *
+     * This method is based on {@link edu.harvard.iq.dataverse.FileDownloadHelper#canDownloadFile(FileMetadata),
+     * and has been adapted to make it callable from the API instead of a view
+     *
+     * @param user requesting the download
+     * @param fileMetadata of the particular file to download
+     * @return boolean
+     */
+    public boolean canDownloadFile(User user, FileMetadata fileMetadata){
+        if (fileMetadata == null){
+            return false;
+        }
+        if ((fileMetadata.getId() == null) || (fileMetadata.getDataFile().getId() == null)){
+            return false;
+        }
+        if (user == null) {
+            return false;
+        }
+        if (user instanceof PrivateUrlUser) {
+            // Always allow download for PrivateUrlUser
+            return true;
+        }
+
+        if (fileMetadata.getDatasetVersion().isDeaccessioned()) {
+            return this.permissionService.userOn(user, fileMetadata.getDatasetVersion().getDataset()).has(Permission.EditDataset);
+        }
+
+        // Note that `isRestricted` at the FileMetadata level is for expressing intent by version. Enforcement is done with `isRestricted` at the DataFile level.
+        boolean isRestrictedFile = fileMetadata.isRestricted() || fileMetadata.getDataFile().isRestricted();
+        if (!isRestrictedFile && !FileUtil.isActivelyEmbargoed(fileMetadata)){
+            return true;
+        }
+
+        // See if the DataverseRequest, which contains IP Groups, has permission to download the file.
+        if (permissionService.requestOn(dvRequestService.getDataverseRequest(), fileMetadata.getDataFile()).has(Permission.DownloadFile)) {
+            logger.fine("The DataverseRequest (User plus IP address) has access to download the file.");
+            return true;
+        }
+
+        return false;
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java
new file mode 100644
index 00000000000..d754dd357a9
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java
@@ -0,0 +1,246 @@
+package edu.harvard.iq.dataverse;
+
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.authorization.users.User;
+import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.time.LocalDate;
+
+import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeAuthenticatedUser;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+@ExtendWith(MockitoExtension.class)
+public class FileDownloadServiceBeanTest {
+
+    @Mock
+    private PermissionServiceBean permissionServiceBeanStub;
+    @Mock
+    private DataverseRequestServiceBean dataverseRequestServiceBeanMock;
+    @Mock
+    private MakeDataCountLoggingServiceBean makeDataCountLoggingServiceBeanMock;
+
+    private FileDownloadServiceBean sut;
+
+    private User testUser;
+
+    @BeforeEach
+    public void setUp() {
+        sut = new FileDownloadServiceBean();
+        sut.permissionService = permissionServiceBeanStub;
+        sut.dvRequestService = dataverseRequestServiceBeanMock;
+        sut.mdcLogService = makeDataCountLoggingServiceBeanMock;
+        testUser = makeAuthenticatedUser("Test", "Test");
+    }
+
+    @Test
+    public void testCanDownloadFile_withoutUser() {
+        assertFalse(sut.canDownloadFile(null, new FileMetadata()));
+    }
+
+    @Test
+    public void testCanDownloadFile_withoutFileMetadata() {
+        assertFalse(sut.canDownloadFile(testUser, null));
+    }
+
+    @Test
+    void testCanDownloadFile_withNullMetadataId() {
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(null);
+
+        assertFalse(sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @Test
+    void testCanDownloadFile_withNullDataFileId() {
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(null);
+        testFileMetadata.setDataFile(testDataFile);
+
+        assertFalse(sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @ParameterizedTest
+    @CsvSource({"false", "true"})
+    void testCanDownloadFile_forDeaccessionedFile(boolean hasPermission) {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setDataset(new Dataset());
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.DEACCESSIONED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+
+        mockPermissionResponseUserOn(hasPermission);
+
+        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @Test
+    void testCanDownloadFile_forUnrestrictedReleasedFile() {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setRestricted(false);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+
+        assertTrue(sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @Test
+    void testCanDownloadFile_forUnrestrictedReleasedActiveEmbargoFile() {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        // With an embargo, an unrestricted file should only be accessible if the embargo has ended
+
+        Embargo testEmbargo = new Embargo(LocalDate.now().plusDays(3), "Still embargoed");
+        testDataFile.setEmbargo(testEmbargo);
+
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setRestricted(false);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+        mockPermissionResponseRequestOn(false);
+
+        assertFalse(sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @Test
+    void testCanDownloadFile_forUnrestrictedReleasedExpiredEmbargoFile() {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        // With an embargo, an unrestricted file should only be accessible if the embargo has ended
+
+        Embargo testEmbargo = new Embargo(LocalDate.now().minusDays(3), "Was embargoed");
+        testDataFile.setEmbargo(testEmbargo);
+
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setRestricted(false);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+
+        assertTrue(sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @ParameterizedTest
+    @CsvSource({"false", "true"})
+    void testCanDownloadFile_forRestrictedReleasedFile(boolean hasPermission) {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setRestricted(true);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+
+        mockPermissionResponseRequestOn(hasPermission);
+
+        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @ParameterizedTest
+    @CsvSource({"false", "true"})
+    void testCanDownloadFile_forRestrictedReleasedFileWithActiveEmbargo(boolean hasPermission) {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        // With an active embargo, a restricted file should have the same access regardless of
+        // embargo state (with an active embargo, there's no way to request permissions,
+        // so the hasPermission=true case primarily applies to the original dataset
+        // creators)
+
+        Embargo testEmbargo = new Embargo(LocalDate.now().plusDays(3), "Still embargoed");
+        testDataFile.setEmbargo(testEmbargo);
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setRestricted(true);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+
+        mockPermissionResponseRequestOn(hasPermission);
+
+        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    @ParameterizedTest
+    @CsvSource({"false", "true"})
+    void testCanDownloadFile_forRestrictedReleasedFileWithExpiredEmbargo(boolean hasPermission) {
+        DataFile testDataFile = new DataFile();
+        testDataFile.setId(2L);
+
+        // With an embargo, a restricted file should have the same access regardless of
+        // embargo state (with an active embargo, there's no way to request permissions,
+        // so the hasPermission=true case primarily applies to the original dataset
+        // creators)
+
+        Embargo testEmbargo = new Embargo(LocalDate.now().minusDays(3), "No longer embargoed");
+        testDataFile.setEmbargo(testEmbargo);
+        DatasetVersion testDatasetVersion = new DatasetVersion();
+        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
+
+        FileMetadata testFileMetadata = new FileMetadata();
+        testFileMetadata.setId(1L);
+        testFileMetadata.setRestricted(true);
+        testFileMetadata.setDataFile(testDataFile);
+        testFileMetadata.setDatasetVersion(testDatasetVersion);
+
+        mockPermissionResponseRequestOn(hasPermission);
+
+        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
+    }
+
+    private void mockPermissionResponseUserOn(boolean response) {
+        PermissionServiceBean.StaticPermissionQuery staticPermissionQueryMock = mock(PermissionServiceBean.StaticPermissionQuery.class);
+
+        when(permissionServiceBeanStub.userOn(ArgumentMatchers.any(), ArgumentMatchers.any(Dataset.class))).thenReturn(staticPermissionQueryMock);
+        when(staticPermissionQueryMock.has(Permission.EditDataset)).thenReturn(response);
+    }
+
+    private void mockPermissionResponseRequestOn(boolean response) {
+        PermissionServiceBean.RequestPermissionQuery requestPermissionQueryMock = mock(PermissionServiceBean.RequestPermissionQuery.class);
+
+        when(permissionServiceBeanStub.requestOn(ArgumentMatchers.any(), ArgumentMatchers.any(DataFile.class))).thenReturn(requestPermissionQueryMock);
+        when(requestPermissionQueryMock.has(Permission.DownloadFile)).thenReturn(response);
+    }
+}

From d5a19f218d82baa377d4dcd26dcb29c4626a9d88 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 6 Jul 2023 16:15:39 -0400
Subject: [PATCH 194/922] add guestbookPopupAction from kebab for request
 access case

---
 src/main/webapp/file-download-button-fragment.xhtml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index 2a8152d5395..a0b79671ae2 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -28,9 +28,10 @@
                        rendered="#{fileDownloadHelper.session.user.authenticated}">
             <p:commandLink styleClass="btn-request #{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user) ? 'italic'  : ''}"
                         actionListener="#{fileDownloadHelper.handleCommandLinkClick(fileMetadata)}"
-                        update="@([id$=requestAccessConsolidated]), @([id$=requestPanel]),  @([id$=filesTable]), @([id$=messagePanel])"
+                        update="@([id$=requestAccessConsolidated]), @([id$=requestPanel]),  @([id$=filesTable]), @([id$=messagePanel]), @([id$=guestbookAndTermsPopup])"
                         id="requestAccessConsolidated"
                         disabled="#{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user)}">
+                 <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle['file.requestAccess'])}"/>
                 #{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']}
             </p:commandLink>
         </p:outputPanel>

From 91fafd7a9b84c8979bb6859d1a29108ab74b8305 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 6 Jul 2023 16:16:03 -0400
Subject: [PATCH 195/922] add todo re: guestbook/no terms case

---
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index a5fb98f7c49..d1e36370d32 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1599,6 +1599,7 @@ public static boolean isRequestAccessPopupRequired(DatasetVersion datasetVersion
         if (answer != null) {
             return answer;
         }
+        //ToDo - also check for guestbook when guestbook at request is true?
         logger.fine("Request access popup is not required.");
         return false;
     }

From 6ead834fe9ca1b7ad1ac1c5d83ff6e156df08255 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 7 Jul 2023 08:55:54 +0100
Subject: [PATCH 196/922] Added: canDataFileBeDownloaded API endpoint

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 15 +++++++--
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 32 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 ++++
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 467341f4077..341fb94e086 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -11,6 +11,7 @@
 import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
 import edu.harvard.iq.dataverse.DataverseServiceBean;
 import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.FileDownloadServiceBean;
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GuestbookResponseServiceBean;
 import edu.harvard.iq.dataverse.TermsOfUseAndAccessValidator;
@@ -106,6 +107,9 @@ public class Files extends AbstractApiBean {
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
 
+    @Inject
+    FileDownloadServiceBean fileDownloadServiceBean;
+
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
     
@@ -824,11 +828,18 @@ public Response getFixityAlgorithm() {
 
     @GET
     @Path("{id}/guestbookResponses/count")
-    public Response getCountGuestbookResponsesByDataFileId(@PathParam("id") String dataFileId) {
+    public Response getCountGuestbookResponsesByDataFileId(@PathParam("id") long dataFileId) {
         try {
-            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(Long.parseLong(dataFileId)).toString());
+            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFileId).toString());
         } catch (NumberFormatException nfe) {
             return badRequest("File identifier has to be numeric.");
         }
     }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/canBeDownloaded")
+    public Response canDataFileBeDownloaded(@Context ContainerRequestContext crc, @PathParam("id") long dataFileId) {
+        return ok(fileDownloadServiceBean.canDownloadFile(getRequestUser(crc), fileSvc.find(dataFileId).getFileMetadata()));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 49b41d1e0e0..033ebe25062 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2054,4 +2054,36 @@ public void testGetCountGuestbookResponsesByDataFileId() {
                 .statusCode(OK.getStatusCode())
                 .body("data.message", equalTo("1"));
     }
+
+    @Test
+    public void testCanDataFileBeDownloaded() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Publish collection and dataset
+        UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+        UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert user can download test file
+        Integer testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response canDataFileBeDownloadedResponse = UtilIT.canDataFileBeDownloaded(testFileId, apiToken);
+
+        canDataFileBeDownloadedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean canDownloadTestFile = JsonPath.from(canDataFileBeDownloadedResponse.body().asString()).getBoolean("data");
+        assertTrue(canDownloadTestFile);
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 6c2543bcf82..4745cc7d2eb 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3275,4 +3275,10 @@ static Response getCountGuestbookResponsesByDataFileId(Integer dataFileId, Strin
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/guestbookResponses/count");
     }
+
+    static Response canDataFileBeDownloaded(Integer dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/files/" + dataFileId + "/canBeDownloaded");
+    }
 }

From fae00a7809db31f8946c65d3c240357bf9273309 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 10:48:11 -0400
Subject: [PATCH 197/922] Handle kebab menu, add resize to popup show request

---
 .../harvard/iq/dataverse/FileDownloadHelper.java    | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index b5f2ee97094..ce2539758ab 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -143,10 +143,15 @@ public void writeGuestbookAndLaunchPackagePopup(GuestbookResponse guestbookRespo
             fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
     }
 
+    
     public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookResponse) {
-            //requestContext.execute("PF('guestbookAndTermsPopup').hide()");
-            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
-            fileDownloadService.writeGuestbookResponseAndRequestAccess(guestbookResponse);
+
+        if(!filesForRequestAccess.isEmpty()) {
+            /* Only for single file requests (i.e. from kebab menu) */
+            guestbookResponse.setDataFile(filesForRequestAccess.get(0));
+        }
+        PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
+        fileDownloadService.writeGuestbookResponseAndRequestAccess(guestbookResponse);
     }
     
      /**
@@ -290,7 +295,7 @@ public void handleCommandLinkClick(FileMetadata fmd){
         
         if (FileUtil.isRequestAccessPopupRequired(fmd.getDatasetVersion())){
             addFileForRequestAccess(fmd.getDataFile());
-            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show()");
+            PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
         } else {
             requestAccess(fmd.getDataFile());
         }

From 892d796a279df87dfb1b6608d5d02aff206f8b0a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 12:20:56 -0400
Subject: [PATCH 198/922] set state for grant and reject

---
 .../edu/harvard/iq/dataverse/DataFile.java    | 17 +++++++++++------
 .../dataverse/ManageFilePermissionsPage.java  | 19 ++++++++-----------
 .../edu/harvard/iq/dataverse/api/Access.java  | 15 +++++++++------
 3 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index 156703d3d0b..ff8c544ddfb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -794,15 +794,20 @@ public void addFileAccessRequester(AuthenticatedUser authenticatedUser) {
         this.fileAccessRequests.add(request);
     }
 
-    public boolean removeFileAccessRequester(RoleAssignee roleAssignee) {
+    public FileAccessRequest getAccessRequestForAssignee(RoleAssignee roleAssignee) {
         if (this.fileAccessRequests == null) {
-            return false;
+            return null;
         }
 
-        FileAccessRequest request = this.fileAccessRequests.stream()
-            .filter(fileAccessRequest -> fileAccessRequest.getRequester().equals(roleAssignee))
-            .findFirst()
-            .orElse(null);
+        return this.fileAccessRequests.stream()
+                .filter(fileAccessRequest -> fileAccessRequest.getRequester().equals(roleAssignee)).findFirst()
+                .orElse(null);
+    }
+
+    public boolean removeFileAccessRequest(FileAccessRequest request) {
+        if (this.fileAccessRequests == null) {
+            return false;
+        }
 
         if (request != null) {
             this.fileAccessRequests.remove(request);
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index 6bf548e65d3..bc9d79eac87 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -458,14 +458,9 @@ private void grantAccessToRequests(AuthenticatedUser au, List<DataFile> files) {
         DataverseRole fileDownloaderRole = roleService.findBuiltinRoleByAlias(DataverseRole.FILE_DOWNLOADER);
         for (DataFile file : files) {
             if (assignRole(au, file, fileDownloaderRole)) {
-                //TODO - why remove requests just to set them again?
-                if (file.removeFileAccessRequester(au)) {
-                    List<FileAccessRequest> fileAccessRequests = fileAccessRequestService.findAll(au.getId(), file.getId(), FileAccessRequest.RequestState.CREATED);
-                    for(FileAccessRequest far : fileAccessRequests){
-                        far.setStateGranted();
-                        fileAccessRequestService.save(far);
-                    }
-                    file.setFileAccessRequests(fileAccessRequests); 
+                FileAccessRequest far = file.getAccessRequestForAssignee(au);
+                if (far!=null) {
+                    far.setStateGranted();
                     datafileService.save(file);
                 }
                 actionPerformed = true;
@@ -497,9 +492,11 @@ public void rejectAccessToAllRequests(AuthenticatedUser au) {
     private void rejectAccessToRequests(AuthenticatedUser au, List<DataFile> files) {
         boolean actionPerformed = false;
         for (DataFile file : files) {
-            if(file.removeFileAccessRequester(au)) {
-            
-                // TODO - set FileAccessRequest.RequestState to REJECTED
+            FileAccessRequest far = file.getAccessRequestForAssignee(au);
+            if(far!=null) {
+                far.setStateRejected();
+                fileAccessRequestService.save(far);
+                file.removeFileAccessRequest(far);
                 datafileService.save(file);
                 actionPerformed = true;
             }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 358b0159e2a..687b859c936 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1535,7 +1535,9 @@ public Response grantFileAccess(@Context ContainerRequestContext crc, @PathParam
 
         try {
             engineSvc.submit(new AssignRoleCommand(ra, fileDownloaderRole, dataFile, dataverseRequest, null));
-            if (dataFile.removeFileAccessRequester(ra)) {
+            FileAccessRequest far = dataFile.getAccessRequestForAssignee(ra);
+            if(far!=null) {
+                far.setStateGranted();
                 dataFileService.save(dataFile);
             }
 
@@ -1660,20 +1662,21 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara
         if (!(dataverseRequest.getAuthenticatedUser().isSuperuser() || permissionService.requestOn(dataverseRequest, dataFile).has(Permission.ManageFilePermissions))) {
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.rejectAccess.failure.noPermissions"));
         }
-
-        if (dataFile.removeFileAccessRequester(ra)) {
+        FileAccessRequest far = dataFile.getAccessRequestForAssignee(ra);
+        if (far != null) {
+            far.setStateRejected();
             dataFileService.save(dataFile);
 
             try {
                 AuthenticatedUser au = (AuthenticatedUser) ra;
-                userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REJECTFILEACCESS, dataFile.getOwner().getId());
+                userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()),
+                        UserNotification.Type.REJECTFILEACCESS, dataFile.getOwner().getId());
             } catch (ClassCastException e) {
-                //nothing to do here - can only send a notification to an authenticated user
+                // nothing to do here - can only send a notification to an authenticated user
             }
 
             List<String> args = Arrays.asList(dataFile.getDisplayName());
             return ok(BundleUtil.getStringFromBundle("access.api.rejectAccess.success.for.single.file", args));
-
         } else {
             List<String> args = Arrays.asList(dataFile.getDisplayName(), ra.getDisplayInfo().getTitle());
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.rejectFailure.noRequest", args));

From ef16a855f14ce2432e0518203033a6dcf587f8dd Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 12:37:07 -0400
Subject: [PATCH 199/922] get requests by state/only get CREATED in perms page

---
 src/main/java/edu/harvard/iq/dataverse/DataFile.java        | 4 ++++
 .../edu/harvard/iq/dataverse/ManageFilePermissionsPage.java | 6 ++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index ff8c544ddfb..daeaf4f55c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -209,6 +209,10 @@ public String toString() {
     public List<FileAccessRequest> getFileAccessRequests(){
         return fileAccessRequests;
     }
+    
+    public List<FileAccessRequest> getFileAccessRequests(FileAccessRequest.RequestState state){
+        return fileAccessRequests.stream().filter(far -> far.getState() == state).collect(Collectors.toList());
+    }
 
     public void setFileAccessRequests(List<FileAccessRequest> fARs){
         this.fileAccessRequests = fARs;
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index bc9d79eac87..ddb853a3db9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -181,16 +181,14 @@ private void initMaps() {
                 fileMap.put(file, raList);
 
                 // populate the file access requests map
-                for (FileAccessRequest fileAccessRequest : file.getFileAccessRequests()) {
+                for (FileAccessRequest fileAccessRequest : file.getFileAccessRequests(FileAccessRequest.RequestState.CREATED)) {
                     List<FileAccessRequest> requestedFiles = fileAccessRequestMap.get(fileAccessRequest.getRequester());
                     if (requestedFiles == null) {
                         requestedFiles = new ArrayList<>();
                         AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getRequester().getUserIdentifier());
                         fileAccessRequestMap.put(withProvider, requestedFiles);
                     }
-                    if(fileAccessRequest.getState() == RequestState.CREATED){
-                        requestedFiles.add(fileAccessRequest);
-                    }
+                    requestedFiles.add(fileAccessRequest);
                 }
             }
         }

From c07e960498e6386f28e7e9528fa8474b858272fe Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 14:15:23 -0400
Subject: [PATCH 200/922] only list unresolved (state=CREATED) access requests

---
 src/main/java/edu/harvard/iq/dataverse/DataFile.java      | 4 ++--
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java   | 2 +-
 .../edu/harvard/iq/dataverse/FileDownloadServiceBean.java | 6 ++----
 src/main/java/edu/harvard/iq/dataverse/api/Access.java    | 2 +-
 src/main/webapp/file-download-button-fragment.xhtml       | 8 ++++----
 src/main/webapp/filesFragment.xhtml                       | 2 +-
 6 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index daeaf4f55c0..4f91e17256c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -821,12 +821,12 @@ public boolean removeFileAccessRequest(FileAccessRequest request) {
         return false;
     }
 
-    public boolean containsFileAccessRequestFromUser(RoleAssignee roleAssignee) {
+    public boolean containsActiveFileAccessRequestFromUser(RoleAssignee roleAssignee) {
         if (this.fileAccessRequests == null) {
             return false;
         }
 
-        Set<AuthenticatedUser> existingUsers = this.fileAccessRequests.stream()
+        Set<AuthenticatedUser> existingUsers = getFileAccessRequests(FileAccessRequest.RequestState.CREATED).stream()
             .map(FileAccessRequest::getRequester)
             .collect(Collectors.toSet());
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index b8020404e91..6ce03ad3a51 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5151,7 +5151,7 @@ public boolean isFileAccessRequestMultiButtonRequired(){
         for (FileMetadata fmd : workingVersion.getFileMetadatas()){
             AuthenticatedUser authenticatedUser = (AuthenticatedUser) session.getUser();
             //Change here so that if all restricted files have pending requests there's no Request Button
-            if ((!this.fileDownloadHelper.canDownloadFile(fmd) && !fmd.getDataFile().containsFileAccessRequestFromUser(authenticatedUser))) {
+            if ((!this.fileDownloadHelper.canDownloadFile(fmd) && !fmd.getDataFile().containsActiveFileAccessRequestFromUser(authenticatedUser))) {
                 return true;
             }
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index f7612300eaf..6341f00d00c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -535,7 +535,7 @@ public boolean requestAccess(Long fileId) {
             return false;
         }
         DataFile file = datafileService.find(fileId);
-        if (!file.containsFileAccessRequestFromUser(session.getUser())) {
+        if (!file.containsActiveFileAccessRequestFromUser(session.getUser())) {
             try {
                 commandEngine.submit(new RequestAccessCommand(dvRequestService.getDataverseRequest(), file));                        
                 return true;
@@ -553,9 +553,7 @@ public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){
             return accessRequested;
         }
 
-        List<AuthenticatedUser> fARs = dataFile.getFileAccessRequesters();
-
-        if(fARs.isEmpty() || (!fARs.isEmpty() && !fARs.contains((AuthenticatedUser)session.getUser()))){
+        if(!dataFile.containsActiveFileAccessRequestFromUser(session.getUser())) {
             try {
                 commandEngine.submit(new RequestAccessCommand(dvRequestService.getDataverseRequest(), dataFile, gbr));
                 accessRequested = true;
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 687b859c936..70b1bdcdd2f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1428,7 +1428,7 @@ public Response requestFileAccess(@Context ContainerRequestContext crc, @PathPar
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.invalidRequest"));
         }
 
-        if (dataFile.containsFileAccessRequestFromUser(requestor)) {
+        if (dataFile.containsActiveFileAccessRequestFromUser(requestor)) {
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.requestExists"));
         }
 
diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index a0b79671ae2..5c29cd6b041 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -24,15 +24,15 @@
         </span></li>
     <ui:fragment rendered="#{!fileMetadata.datasetVersion.deaccessioned and !fileDownloadHelper.canDownloadFile(fileMetadata)
                              and fileMetadata.dataFile.owner.fileAccessRequest and !dataFileServiceBean.isActivelyEmbargoed(fileMetadata)}">
-        <p:outputPanel id="requestPanel" styleClass="iq-dropdown-list-item #{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user) ? 'disabled' : ''}"
+        <p:outputPanel id="requestPanel" styleClass="iq-dropdown-list-item #{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user) ? 'disabled' : ''}"
                        rendered="#{fileDownloadHelper.session.user.authenticated}">
-            <p:commandLink styleClass="btn-request #{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user) ? 'italic'  : ''}"
+            <p:commandLink styleClass="btn-request #{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user) ? 'italic'  : ''}"
                         actionListener="#{fileDownloadHelper.handleCommandLinkClick(fileMetadata)}"
                         update="@([id$=requestAccessConsolidated]), @([id$=requestPanel]),  @([id$=filesTable]), @([id$=messagePanel]), @([id$=guestbookAndTermsPopup])"
                         id="requestAccessConsolidated"
-                        disabled="#{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user)}">
+                        disabled="#{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user)}">
                  <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle['file.requestAccess'])}"/>
-                #{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']}
+                #{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']}
             </p:commandLink>
         </p:outputPanel>
         <li jsf:rendered="#{!fileDownloadHelper.session.user.authenticated}">
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 7e1cb4ac4cd..5f2c15df048 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -514,7 +514,7 @@
                        data-content="#{DatasetPage.ingestMessage} #{fileMetadata.dataFile.ingestReportMessage}"/>
                 </span>
             </div>
-            <div class="bg-info text-info text-center margin-bottom-half" jsf:rendered="#{fileMetadata.dataFile.containsFileAccessRequestFromUser(dataverseSession.user)}">
+            <div class="bg-info text-info text-center margin-bottom-half" jsf:rendered="#{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user)}">
                 <!-- Access Requested -->
                 <span class="glyphicon glyphicon-info-sign"/>
                 <span> #{bundle['file.accessRequested']}&#160;</span>

From 60165813192a3582557db840aa941d93217b8494 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 14:19:19 -0400
Subject: [PATCH 201/922] fix bug from #9257 which changed return type to FAR
 from DataFile

seen in the Edit Permissions/File dialog when clicking the link with the
# of files listed. In that dialog, things explode trying to get the
directoryLabel of a FileAccessRequest (seen in log) and the popup dialog
incorrectly shows the input to select a user, only has a grant button
(no reject), and doesn't work (the last could be an issue with this
branch only).
---
 .../dataverse/ManageFilePermissionsPage.java  | 21 ++++++++++++++-----
 .../webapp/permissions-manage-files.xhtml     |  2 +-
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index ddb853a3db9..e8e9ae9f167 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -29,6 +29,8 @@
 import java.util.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
 import javax.ejb.EJB;
 import javax.faces.application.FacesMessage;
 import javax.faces.event.ActionEvent;
@@ -89,6 +91,15 @@ public class ManageFilePermissionsPage implements java.io.Serializable {
     public TreeMap<AuthenticatedUser, List<FileAccessRequest>> getFileAccessRequestMap() {
         return fileAccessRequestMap;
     }
+    
+    public List<DataFile> getDataFilesForRequestor() {
+        List<FileAccessRequest> fars = fileAccessRequestMap.get(getFileRequester());
+        if (fars == null) {
+            return new ArrayList<>();
+        } else {
+            return fars.stream().map(FileAccessRequest::getDataFile).collect(Collectors.toList());
+        }
+    }
 
     private final TreeMap<AuthenticatedUser,List<FileAccessRequest>> fileAccessRequestMap = new TreeMap<>();
     private boolean showDeleted = true;
@@ -182,13 +193,13 @@ private void initMaps() {
 
                 // populate the file access requests map
                 for (FileAccessRequest fileAccessRequest : file.getFileAccessRequests(FileAccessRequest.RequestState.CREATED)) {
-                    List<FileAccessRequest> requestedFiles = fileAccessRequestMap.get(fileAccessRequest.getRequester());
-                    if (requestedFiles == null) {
-                        requestedFiles = new ArrayList<>();
+                    List<FileAccessRequest> fileAccessRequestList = fileAccessRequestMap.get(fileAccessRequest.getRequester());
+                    if (fileAccessRequestList == null) {
+                        fileAccessRequestList = new ArrayList<>();
                         AuthenticatedUser withProvider = authenticationService.getAuthenticatedUserWithProvider(fileAccessRequest.getRequester().getUserIdentifier());
-                        fileAccessRequestMap.put(withProvider, requestedFiles);
+                        fileAccessRequestMap.put(withProvider, fileAccessRequestList);
                     }
-                    requestedFiles.add(fileAccessRequest);
+                    fileAccessRequestList.add(fileAccessRequest);
                 }
             }
         }
diff --git a/src/main/webapp/permissions-manage-files.xhtml b/src/main/webapp/permissions-manage-files.xhtml
index d3109da69a6..4e4e56f2051 100644
--- a/src/main/webapp/permissions-manage-files.xhtml
+++ b/src/main/webapp/permissions-manage-files.xhtml
@@ -322,7 +322,7 @@
                                 <o:importFunctions type="java.util.Arrays" />
                                 <p:dataTable id="restrictedFilesForAccess" var="file"
                                              value="#{empty manageFilePermissionsPage.fileRequester ? Arrays:asList(manageFilePermissionsPage.fileMap.keySet().toArray()) :
-                                                      manageFilePermissionsPage.fileAccessRequestMap.get(manageFilePermissionsPage.fileRequester)}"
+                                                      manageFilePermissionsPage.dataFilesForRequestor}"
                                              selection="#{manageFilePermissionsPage.selectedFiles}" rowKey="#{file.id}" sortBy="#{file.displayName}"
                                              scrollable="true" scrollHeight="300">
 

From 0720c93fd7b3e1728824a24e36c3e27a8db65509 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 15:03:02 -0400
Subject: [PATCH 202/922] fix grant in popup from # files link in Files column

---
 .../java/edu/harvard/iq/dataverse/DataFile.java    |  2 +-
 .../iq/dataverse/ManageFilePermissionsPage.java    | 14 ++++----------
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index 4f91e17256c..7b59487d3f3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -804,7 +804,7 @@ public FileAccessRequest getAccessRequestForAssignee(RoleAssignee roleAssignee)
         }
 
         return this.fileAccessRequests.stream()
-                .filter(fileAccessRequest -> fileAccessRequest.getRequester().equals(roleAssignee)).findFirst()
+                .filter(fileAccessRequest -> fileAccessRequest.getRequester().equals(roleAssignee) && fileAccessRequest.isStateCreated()).findFirst()
                 .orElse(null);
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
index e8e9ae9f167..4796d0956c6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManageFilePermissionsPage.java
@@ -421,18 +421,12 @@ public void grantAccess(ActionEvent evt) {
                     if (file.isReleased()) {
                         sendNotification = true;
                     }
-                    // remove request, if it exist
+                    // set request(s) granted, if they exist
                     for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
-                            if (file.getFileAccessRequesters().remove(au)) {
-                                List<FileAccessRequest> fileAccessRequests = fileAccessRequestService.findAllByAuthenticatedUserIdAndRequestState(au.getId(), FileAccessRequest.RequestState.CREATED);
-                                for(FileAccessRequest far : fileAccessRequests){
-                                    far.setStateGranted();
-                                    fileAccessRequestService.save(far);
-                                }
-                                file.setFileAccessRequests(fileAccessRequests); 
-                                datafileService.save(file);
-                            }       
+                        FileAccessRequest far = file.getAccessRequestForAssignee(au);
+                        far.setStateGranted();
                     }
+                    datafileService.save(file);
                 }
 
             }

From 267a5ccb7bd4c9691a61bddb48e60e8067550656 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 15:03:53 -0400
Subject: [PATCH 203/922] only require one CREATED request per datafile, auth
 user

and allow multiple granted/rejected ones
---
 src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java | 4 +---
 .../db/migration/V5.13.0.3__guestbook-on-request.sql          | 2 ++
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
index e76a3414b52..a9be3ecac5e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
@@ -27,9 +27,7 @@
  */
 
 @Entity
-@Table(name = "fileaccessrequests", //having added the guestbookresponse_id column to fileaccessrequests
-    uniqueConstraints=@UniqueConstraint(columnNames={"datafile_id", "authenticated_user_id","request_state"}) //this may not make sense at some future point
-)
+@Table(name = "fileaccessrequests")
 
 @NamedQueries({
         @NamedQuery(name = "FileAccessRequest.findByAuthenticatedUserId",
diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 20bc1fc1c76..166fb9554c6 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -5,3 +5,5 @@ ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KE
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_guestbookresponse;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);
+ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS created_requests;
+CREATE UNIQUE INDEX created_requests ON fileaccessrequests (datafile_id, authenticated_user_id) WHERE request_state='CREATED';

From 4ba20d606e6f595b97f5960386130f7af6acfb76 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 15:05:42 -0400
Subject: [PATCH 204/922] simplify states

---
 .../iq/dataverse/FileAccessRequest.java       | 47 +------------------
 1 file changed, 1 insertion(+), 46 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
index a9be3ecac5e..ddde8049dff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
@@ -67,7 +67,7 @@ public class FileAccessRequest implements Serializable{
     @JoinColumn(nullable=true)
     private GuestbookResponse guestbookResponse;
     
-    public enum RequestState {CREATED,EDITED,GRANTED,REJECTED,RESUBMIT,INVALIDATED,CLOSED};
+    public enum RequestState {CREATED, GRANTED, REJECTED};
     //private RequestState state;
     @Enumerated(EnumType.STRING)
     @Column(name="request_state", nullable=false )
@@ -138,24 +138,12 @@ public String getStateLabel() {
         if(isStateCreated()){
             return "created";
         }
-        if(isStateEdited()) {
-            return "edited";
-        }
         if(isStateGranted()) {
             return "granted";
         }
         if(isStateRejected()) {
             return "rejected";
         }
-        if(isStateResubmit()) {
-            return "resubmit";
-        }
-        if(isStateInvalidated()) {
-            return "invalidated";
-        }
-        if(isStateClosed()) {
-            return "closed";
-        }
         return null; 
     }
     
@@ -163,10 +151,6 @@ public void setStateCreated() {
         this.requestState = RequestState.CREATED;
     }
     
-    public void setStateEdited() {
-        this.requestState = RequestState.EDITED;
-    }
-    
     public void setStateGranted() {
         this.requestState = RequestState.GRANTED;
     }
@@ -175,27 +159,10 @@ public void setStateRejected() {
         this.requestState = RequestState.REJECTED;
     }
 
-    public void setStateResubmit() {
-        this.requestState = RequestState.RESUBMIT;
-    }
-    
-    public void setStateInvalidated() {
-        this.requestState = RequestState.INVALIDATED;
-    }
-
-    public void setStateClosed() {
-        this.requestState = RequestState.CLOSED;
-    }
-
-    
     public boolean isStateCreated() {
         return this.requestState == RequestState.CREATED;
     }
    
-    public boolean isStateEdited() {
-        return this.requestState == RequestState.EDITED;
-    }
-    
     public boolean isStateGranted() {
         return this.requestState == RequestState.GRANTED;
     }
@@ -203,18 +170,6 @@ public boolean isStateGranted() {
     public boolean isStateRejected() {
         return this.requestState == RequestState.REJECTED;
     }
-
-    public boolean isStateResubmit() {
-        return this.requestState == RequestState.RESUBMIT;
-    }
-    
-    public boolean isStateInvalidated() {
-        return this.requestState == RequestState.INVALIDATED;
-    }
-
-    public boolean isStateClosed() {
-        return this.requestState == RequestState.CLOSED;
-    }
     
     @Override
     public int hashCode() {

From e1e075cf7961d922d35c89eb9f719fa0451ba56d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 16:25:03 -0400
Subject: [PATCH 205/922] typo in comment

---
 src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index ce2539758ab..7ec8272ac5b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -310,7 +310,7 @@ public void requestAccessMultiple(List<DataFile> files) {
          DataFile notificationFile = null;
          for (DataFile file : files) {
              //Not sending notification via request method so that
-             // we can bundle them up into one nofication at dataset level
+             // we can bundle them up into one notification at dataset level
              test = processRequestAccess(file, false);
              succeeded |= test;
              if (notificationFile == null) {

From f980d7ba1f3ad395ffbb21a27a04e2e41433a7f7 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 16:25:23 -0400
Subject: [PATCH 206/922] Fix idempotence

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index 166fb9554c6..a92d8910f6d 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -5,5 +5,5 @@ ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KE
 ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT;
 ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_guestbookresponse;
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);
-ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS created_requests;
+DROP INDEX IF EXISTS created_requests;
 CREATE UNIQUE INDEX created_requests ON fileaccessrequests (datafile_id, authenticated_user_id) WHERE request_state='CREATED';

From 6a0b0d0551926d56e1c80b9e1d9681984a13a98d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 7 Jul 2023 16:26:04 -0400
Subject: [PATCH 207/922] ignore prior granted/rejected requests

fixes multi-file request access button on dataset page
---
 .../iq/dataverse/authorization/users/AuthenticatedUser.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java
index bb688fb8acb..3a7bbaf5af9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/AuthenticatedUser.java
@@ -222,7 +222,9 @@ public List<DataFile> getRequestedDataFiles(){
         List<DataFile> requestedDataFiles = new ArrayList<>();
 
         for(FileAccessRequest far : getFileAccessRequests()){
-            requestedDataFiles.add(far.getDataFile());
+            if(far.isStateCreated()) {
+                requestedDataFiles.add(far.getDataFile());
+            }
         }
         return requestedDataFiles;
     }

From 9f35bf7843f1dc810e77217aec7bc1fe5dab17ad Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Sun, 9 Jul 2023 15:35:49 +0100
Subject: [PATCH 208/922] Added: naming refactor and managing not found files
 in new files API endpoints

---
 .../iq/dataverse/FileDownloadServiceBean.java |  1 -
 .../edu/harvard/iq/dataverse/api/Files.java   | 21 ++++++++++++-------
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 20 +++++++++---------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  8 ++-----
 4 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 5c490b18ecc..f5bb60510cc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -574,7 +574,6 @@ public String getDirectStorageLocatrion(String storageLocation) {
     }
 
     /**
-     *
      * Checks if a user can download a file based on the file metadata and the permissions of the user
      *
      * This method is based on {@link edu.harvard.iq.dataverse.FileDownloadHelper#canDownloadFile(FileMetadata),
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 341fb94e086..d4bf28482c9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -106,7 +106,6 @@ public class Files extends AbstractApiBean {
     MakeDataCountLoggingServiceBean mdcLogService;
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
-
     @Inject
     FileDownloadServiceBean fileDownloadServiceBean;
 
@@ -828,18 +827,26 @@ public Response getFixityAlgorithm() {
 
     @GET
     @Path("{id}/guestbookResponses/count")
-    public Response getCountGuestbookResponsesByDataFileId(@PathParam("id") long dataFileId) {
+    public Response getCountGuestbookResponses(@PathParam("id") String dataFileId) {
+        DataFile dataFile;
         try {
-            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFileId).toString());
-        } catch (NumberFormatException nfe) {
-            return badRequest("File identifier has to be numeric.");
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
         }
+        return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
     }
 
     @GET
     @AuthRequired
     @Path("{id}/canBeDownloaded")
-    public Response canDataFileBeDownloaded(@Context ContainerRequestContext crc, @PathParam("id") long dataFileId) {
-        return ok(fileDownloadServiceBean.canDownloadFile(getRequestUser(crc), fileSvc.find(dataFileId).getFileMetadata()));
+    public Response canFileBeDownloaded(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        return ok(fileDownloadServiceBean.canDownloadFile(getRequestUser(crc), dataFile.getFileMetadata()));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 033ebe25062..8cdc52be1d0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2020,7 +2020,7 @@ public void testDeleteFile() {
     }
 
     @Test
-    public void testGetCountGuestbookResponsesByDataFileId() {
+    public void testGetCountGuestbookResponses() {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -2043,20 +2043,20 @@ public void testGetCountGuestbookResponsesByDataFileId() {
         UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
 
         // Download test file
-        Integer testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
 
         Response downloadResponse = UtilIT.downloadFile(testFileId, apiToken);
         downloadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // Get count guestbook responses and assert it is 1
-        Response getGuestbookResponsesByDataFileIdResponse = UtilIT.getCountGuestbookResponsesByDataFileId(testFileId, apiToken);
-        getGuestbookResponsesByDataFileIdResponse.then().assertThat()
+        Response getCountGuestbookResponsesResponse = UtilIT.getCountGuestbookResponses(testFileId, apiToken);
+        getCountGuestbookResponsesResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data.message", equalTo("1"));
     }
 
     @Test
-    public void testCanDataFileBeDownloaded() {
+    public void testCanFileBeDownloaded() {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -2079,11 +2079,11 @@ public void testCanDataFileBeDownloaded() {
         UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
 
         // Assert user can download test file
-        Integer testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response canDataFileBeDownloadedResponse = UtilIT.canDataFileBeDownloaded(testFileId, apiToken);
+        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response canFileBeDownloadedResponse = UtilIT.canFileBeDownloaded(testFileId, apiToken);
 
-        canDataFileBeDownloadedResponse.then().assertThat().statusCode(OK.getStatusCode());
-        boolean canDownloadTestFile = JsonPath.from(canDataFileBeDownloadedResponse.body().asString()).getBoolean("data");
-        assertTrue(canDownloadTestFile);
+        canFileBeDownloadedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean canFileBeDownloaded = JsonPath.from(canFileBeDownloadedResponse.body().asString()).getBoolean("data");
+        assertTrue(canFileBeDownloaded);
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4745cc7d2eb..4f2ab3146ef 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -1,6 +1,5 @@
 package edu.harvard.iq.dataverse.api;
 
-import com.jayway.restassured.RestAssured;
 import com.jayway.restassured.http.ContentType;
 import com.jayway.restassured.path.json.JsonPath;
 import com.jayway.restassured.response.Response;
@@ -12,8 +11,6 @@
 import javax.json.JsonObjectBuilder;
 import javax.json.JsonArrayBuilder;
 import javax.json.JsonObject;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -44,7 +41,6 @@
 import static com.jayway.restassured.path.xml.XmlPath.from;
 import static com.jayway.restassured.RestAssured.given;
 import edu.harvard.iq.dataverse.DatasetField;
-import edu.harvard.iq.dataverse.DatasetFieldConstant;
 import edu.harvard.iq.dataverse.DatasetFieldType;
 import edu.harvard.iq.dataverse.DatasetFieldValue;
 import edu.harvard.iq.dataverse.util.StringUtil;
@@ -3270,13 +3266,13 @@ static Response createAndUploadTestFile(String persistentId, String testFileName
         return uploadZipFileViaSword(persistentId, pathToTestFile, apiToken);
     }
 
-    static Response getCountGuestbookResponsesByDataFileId(Integer dataFileId, String apiToken) {
+    static Response getCountGuestbookResponses(int dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/guestbookResponses/count");
     }
 
-    static Response canDataFileBeDownloaded(Integer dataFileId, String apiToken) {
+    static Response canFileBeDownloaded(int dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/canBeDownloaded");

From a2bc4d4b1d98cb103706c49e1c357a3407b6c23b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Sun, 9 Jul 2023 15:47:08 +0100
Subject: [PATCH 209/922] Removed: not essential findDataFileOrDie call to
 avoid extra query

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index d4bf28482c9..714a08f4d2c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -827,14 +827,8 @@ public Response getFixityAlgorithm() {
 
     @GET
     @Path("{id}/guestbookResponses/count")
-    public Response getCountGuestbookResponses(@PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
+    public Response getCountGuestbookResponses(@PathParam("id") long dataFileId) {
+        return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFileId).toString());
     }
 
     @GET

From ec755348740e33b30f4909e78eee2e9f3c63f8a9 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 10 Jul 2023 09:11:22 +0100
Subject: [PATCH 210/922] Added: getFileThumbnailClass API endpoint and
 enhanced test coverage for new endpoints

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 15 ++++++
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 51 ++++++++++++++++---
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  8 ++-
 3 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 714a08f4d2c..2f5699d28ca 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -3,6 +3,7 @@
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetLock;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
@@ -108,6 +109,8 @@ public class Files extends AbstractApiBean {
     GuestbookResponseServiceBean guestbookResponseService;
     @Inject
     FileDownloadServiceBean fileDownloadServiceBean;
+    @Inject
+    DataFileServiceBean dataFileServiceBean;
 
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
@@ -843,4 +846,16 @@ public Response canFileBeDownloaded(@Context ContainerRequestContext crc, @PathP
         }
         return ok(fileDownloadServiceBean.canDownloadFile(getRequestUser(crc), dataFile.getFileMetadata()));
     }
+
+    @GET
+    @Path("{id}/thumbnailClass")
+    public Response getFileThumbnailClass(@PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        return ok(dataFileServiceBean.getFileThumbnailClass(dataFile));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 8cdc52be1d0..b3b96dc5234 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2020,7 +2020,7 @@ public void testDeleteFile() {
     }
 
     @Test
-    public void testGetCountGuestbookResponses() {
+    public void testGetCountGuestbookResponses() throws InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -2048,6 +2048,9 @@ public void testGetCountGuestbookResponses() {
         Response downloadResponse = UtilIT.downloadFile(testFileId, apiToken);
         downloadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
+        // Ensure guestbook is updated
+        sleep(2000);
+
         // Get count guestbook responses and assert it is 1
         Response getCountGuestbookResponsesResponse = UtilIT.getCountGuestbookResponses(testFileId, apiToken);
         getCountGuestbookResponsesResponse.then().assertThat()
@@ -2067,23 +2070,55 @@ public void testCanFileBeDownloaded() {
 
         Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
         // Upload test file
         String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
         uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Publish collection and dataset
-        UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
-        UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
-
         // Assert user can download test file
         int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response canFileBeDownloadedResponse = UtilIT.canFileBeDownloaded(testFileId, apiToken);
+        Response canFileBeDownloadedResponse = UtilIT.canFileBeDownloaded(Integer.toString(testFileId), apiToken);
 
         canFileBeDownloadedResponse.then().assertThat().statusCode(OK.getStatusCode());
         boolean canFileBeDownloaded = JsonPath.from(canFileBeDownloadedResponse.body().asString()).getBoolean("data");
         assertTrue(canFileBeDownloaded);
+
+        // Call with invalid file id
+        Response canFileBeDownloadedInvalidIdResponse = UtilIT.canFileBeDownloaded("testInvalidId", apiToken);
+        canFileBeDownloadedInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+    }
+
+    @Test
+    public void testGetFileThumbnailClass() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get file thumbnail class and assert is image
+        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response getFileThumbnailClassResponse = UtilIT.getFileThumbnailClass(Integer.toString(testFileId), apiToken);
+
+        getFileThumbnailClassResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String fileThumbnailClass = JsonPath.from(getFileThumbnailClassResponse.body().asString()).getString("data.message");
+        assertEquals("image", fileThumbnailClass);
+
+        // Call with invalid file id
+        Response getFileThumbnailClassInvalidIdResponse = UtilIT.getFileThumbnailClass("testInvalidId", apiToken);
+        getFileThumbnailClassInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4f2ab3146ef..502a01f0d32 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3272,9 +3272,15 @@ static Response getCountGuestbookResponses(int dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/guestbookResponses/count");
     }
 
-    static Response canFileBeDownloaded(int dataFileId, String apiToken) {
+    static Response canFileBeDownloaded(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/canBeDownloaded");
     }
+
+    static Response getFileThumbnailClass(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/files/" + dataFileId + "/thumbnailClass");
+    }
 }

From 86865f5a979625e1f948c20d0618024f792e2c56 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 10 Jul 2023 12:48:08 +0100
Subject: [PATCH 211/922] Added: getCountGuestbookResponses PIDs support and
 param format and data file existence verifications

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java  | 10 ++++++++--
 .../java/edu/harvard/iq/dataverse/api/FilesIT.java     |  6 +++++-
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java |  2 +-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 2f5699d28ca..171f8aa6e87 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -830,8 +830,14 @@ public Response getFixityAlgorithm() {
 
     @GET
     @Path("{id}/guestbookResponses/count")
-    public Response getCountGuestbookResponses(@PathParam("id") long dataFileId) {
-        return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFileId).toString());
+    public Response getCountGuestbookResponses(@PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
     }
 
     @GET
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index b3b96dc5234..fe2007a37e4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2052,10 +2052,14 @@ public void testGetCountGuestbookResponses() throws InterruptedException {
         sleep(2000);
 
         // Get count guestbook responses and assert it is 1
-        Response getCountGuestbookResponsesResponse = UtilIT.getCountGuestbookResponses(testFileId, apiToken);
+        Response getCountGuestbookResponsesResponse = UtilIT.getCountGuestbookResponses(Integer.toString(testFileId), apiToken);
         getCountGuestbookResponsesResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data.message", equalTo("1"));
+
+        // Call with invalid file id
+        Response getCountGuestbookResponsesInvalidIdResponse = UtilIT.getCountGuestbookResponses("testInvalidId", apiToken);
+        getCountGuestbookResponsesInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 502a01f0d32..9a471648da4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3266,7 +3266,7 @@ static Response createAndUploadTestFile(String persistentId, String testFileName
         return uploadZipFileViaSword(persistentId, pathToTestFile, apiToken);
     }
 
-    static Response getCountGuestbookResponses(int dataFileId, String apiToken) {
+    static Response getCountGuestbookResponses(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/guestbookResponses/count");

From 5cfcac1874a44b9c8355b3025cfecdd101472fbc Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 10 Jul 2023 10:10:45 -0400
Subject: [PATCH 212/922] make FileDownloadHelper visible in file.xhtml

---
 src/main/java/edu/harvard/iq/dataverse/FilePage.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
index f64a38d46d7..70faeeac815 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
@@ -1199,5 +1199,14 @@ public String getIngestMessage() {
     public boolean isHasPublicStore() {
         return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.PublicInstall, StorageIO.isPublicStore(DataAccess.getStorageDriverFromIdentifier(file.getStorageIdentifier())));
     }
+    
+    //Allows use of fileDownloadHelper in file.xhtml
+    public FileDownloadHelper getFileDownloadHelper() {
+        return fileDownloadHelper;
+    }
+
+    public void setFileDownloadHelper(FileDownloadHelper fileDownloadHelper) {
+        this.fileDownloadHelper = fileDownloadHelper;
+    }
 
 }

From e87074a9ecaaf75513ecf1add69d6022cbaac36d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 11 Jul 2023 14:38:26 -0400
Subject: [PATCH 213/922] add default setting

---
 .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java     | 1 +
 src/main/resources/META-INF/microprofile-config.properties      | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index af325bfa426..2386dd70273 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -49,6 +49,7 @@ public enum JvmSettings {
     // FILES SETTINGS
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
+    GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbookAtRequest"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 7c16495f870..09a3bbaae51 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -10,6 +10,8 @@ dataverse.build=
 
 # FILES
 dataverse.files.directory=/tmp/dataverse
+#Default to original behavior - guestbook appears when UI user starts a download (after they've requested and been given access)
+dataverse.files.guestbookAtRequest=false
 # The variables are replaced with the environment variables from our base image, but still easy to override
 %ct.dataverse.files.directory=${STORAGE_DIR}
 %ct.dataverse.files.uploads=${STORAGE_DIR}/uploads

From f1b81745d30b63c63e00c3bc9e4f930db518bdfe Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 11 Jul 2023 14:38:34 -0400
Subject: [PATCH 214/922] comment typo

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 6ce03ad3a51..2a8a848b401 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5261,7 +5261,7 @@ public boolean isTermsPopupRequired(){
     }
     
     public boolean isGuestbookPopupRequiredAtDownload(){
-        return isGuestbookPopupRequired() && !workingVersion.getDataset().isFileAccessRequest(); //only show guestbookAtDwonload if there is no possible request access
+        return isGuestbookPopupRequired() && !workingVersion.getDataset().isFileAccessRequest(); //only show guestbookAtDownload if there is no possible request access
     }
     
     public String requestAccessMultipleFiles() {

From 8ead509cd9c27941585ce7eba6ae81edbf9071b5 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 11 Jul 2023 15:18:02 -0400
Subject: [PATCH 215/922] add setting to isGB popup req at download, remove mp
 cofig default

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java    | 5 ++++-
 src/main/resources/META-INF/microprofile-config.properties | 2 --
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 2a8a848b401..8241b74f557 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5261,7 +5261,10 @@ public boolean isTermsPopupRequired(){
     }
     
     public boolean isGuestbookPopupRequiredAtDownload(){
-        return isGuestbookPopupRequired() && !workingVersion.getDataset().isFileAccessRequest(); //only show guestbookAtDownload if there is no possible request access
+        // Only show guestbookAtDownload if fileAccessRequest is turned off for the
+        // dataset or guestbook at request is disabled (legacy behavior)
+        return isGuestbookPopupRequired() && (!workingVersion.getDataset().isFileAccessRequest()
+                || JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class).orElse(false));
     }
     
     public String requestAccessMultipleFiles() {
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 09a3bbaae51..7c16495f870 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -10,8 +10,6 @@ dataverse.build=
 
 # FILES
 dataverse.files.directory=/tmp/dataverse
-#Default to original behavior - guestbook appears when UI user starts a download (after they've requested and been given access)
-dataverse.files.guestbookAtRequest=false
 # The variables are replaced with the environment variables from our base image, but still easy to override
 %ct.dataverse.files.directory=${STORAGE_DIR}
 %ct.dataverse.files.uploads=${STORAGE_DIR}/uploads

From 5f76d8119cffc55d961269f6745b4b0fbd6b1a1e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 11 Jul 2023 16:47:22 -0400
Subject: [PATCH 216/922] fix logic, add to gb popup

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 src/main/webapp/guestbook-terms-popup-fragment.xhtml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 8241b74f557..1dd832a5815 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5264,7 +5264,7 @@ public boolean isGuestbookPopupRequiredAtDownload(){
         // Only show guestbookAtDownload if fileAccessRequest is turned off for the
         // dataset or guestbook at request is disabled (legacy behavior)
         return isGuestbookPopupRequired() && (!workingVersion.getDataset().isFileAccessRequest()
-                || JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class).orElse(false));
+                || !(JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class).orElse(false)));
     }
     
     public String requestAccessMultipleFiles() {
diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 4bddbb0dd38..4099d6faa1f 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -141,7 +141,7 @@
                 value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}"
                 escape="false" />
         </div>
-            <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((DatasetPage.termsGuestbookPopupAction != bundle.download) or DatasetPage.guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
+            <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((DatasetPage.termsGuestbookPopupAction != bundle.download) != DatasetPage.guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
                 <!--
                 Only validate the active context. For example, the user might be filling out
                 the guestbook on the Preview tab rather than after clicking the "download file"

From 9e88d7d39c591943760db193d496d9e158026541 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 10:00:59 -0400
Subject: [PATCH 217/922] initial attempt at per-collection setting

---
 .../edu/harvard/iq/dataverse/DatasetPage.java |  4 +-
 .../iq/dataverse/DvObjectContainer.java       | 48 ++++++++++++++++---
 .../harvard/iq/dataverse/SettingsWrapper.java | 34 +++++++++++--
 .../iq/dataverse/dataverse/DataverseUtil.java |  6 +--
 src/main/java/propertyFiles/Bundle.properties |  2 +
 src/main/webapp/dataverse.xhtml               | 17 +++++++
 .../dataverse/DataverseUtilTest.java          |  4 +-
 7 files changed, 98 insertions(+), 17 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 1dd832a5815..b2e26a35fc2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5899,7 +5899,7 @@ public String getEffectiveMetadataLanguage() {
     }
     public String getEffectiveMetadataLanguage(boolean ofParent) {
         String mdLang = ofParent ? dataset.getOwner().getEffectiveMetadataLanguage() : dataset.getEffectiveMetadataLanguage();
-        if (mdLang.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) {
+        if (mdLang.equals(DvObjectContainer.UNDEFINED_CODE)) {
             mdLang = settingsWrapper.getDefaultMetadataLanguage();
         }
         return mdLang;
@@ -5907,7 +5907,7 @@ public String getEffectiveMetadataLanguage(boolean ofParent) {
 
     public String getLocaleDisplayName(String code) {
         String displayName = settingsWrapper.getBaseMetadataLanguageMap(false).get(code);
-        if(displayName==null && !code.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) {
+        if(displayName==null && !code.equals(DvObjectContainer.UNDEFINED_CODE)) {
             //Default (for cases such as :when a Dataset has a metadatalanguage code but :MetadataLanguages is no longer defined).
             displayName = new Locale(code).getDisplayName();
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index 6ff01ef3ea8..7d8a741b267 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -1,8 +1,11 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.util.Locale;
+import java.util.Optional;
+
 import javax.persistence.MappedSuperclass;
 import org.apache.commons.lang3.StringUtils;
 
@@ -13,10 +16,8 @@
  */
 @MappedSuperclass
 public abstract class DvObjectContainer extends DvObject {
-	
-    
-    public static final String UNDEFINED_METADATA_LANGUAGE_CODE = "undefined"; //Used in dataverse.xhtml as a non-null selection option value (indicating inheriting the default)
     
+    public static final String UNDEFINED_CODE = "undefined"; //Used in dataverse.xhtml as a non-null selection option value (indicating inheriting the default)
     
     public void setOwner(Dataverse owner) {
         super.setOwner(owner);
@@ -38,6 +39,8 @@ public boolean isEffectivelyPermissionRoot() {
     
     private String metadataLanguage=null;
     
+    private String guestbookAtRequest = null;
+    
     public String getEffectiveStorageDriverId() {
         String id = storageDriver;
         if (StringUtils.isBlank(id)) {
@@ -71,7 +74,7 @@ public String getEffectiveMetadataLanguage() {
             if (this.getOwner() != null) {
                 ml = this.getOwner().getEffectiveMetadataLanguage();
             } else {
-                ml = UNDEFINED_METADATA_LANGUAGE_CODE;
+                ml = UNDEFINED_CODE;
             }
         }
         return ml;
@@ -79,13 +82,13 @@ public String getEffectiveMetadataLanguage() {
     
     public String getMetadataLanguage() {
         if (metadataLanguage == null) {
-            return UNDEFINED_METADATA_LANGUAGE_CODE;
+            return UNDEFINED_CODE;
         }
         return metadataLanguage;
     }
 
     public void setMetadataLanguage(String ml) {
-        if (ml != null && ml.equals(UNDEFINED_METADATA_LANGUAGE_CODE)) {
+        if (ml != null && ml.equals(UNDEFINED_CODE)) {
             this.metadataLanguage = null;
         } else {
             this.metadataLanguage = ml;
@@ -93,7 +96,38 @@ public void setMetadataLanguage(String ml) {
     }
     
     public static boolean isMetadataLanguageSet(String mdLang) {
-        return mdLang!=null && !mdLang.equals(UNDEFINED_METADATA_LANGUAGE_CODE);
+        return mdLang!=null && !mdLang.equals(UNDEFINED_CODE);
+    }
+    
+    public boolean getEffectiveGuestbookEntryAtRequest() {
+        boolean gbAtRequest = false;
+        if (guestbookAtRequest==null || UNDEFINED_CODE.equals(guestbookAtRequest)) {
+            if (this.getOwner() != null) {
+                gbAtRequest = this.getOwner().getEffectiveGuestbookEntryAtRequest();
+            } else {
+                Optional<Boolean> opt = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class);
+                if (opt.isPresent()) {
+                gbAtRequest = opt.get();
+                }
+            }
+        }
+        return gbAtRequest;
+    }
+    
+    public String getGuestbookEntryAtRequest() {
+        if (guestbookAtRequest == null) {
+            return UNDEFINED_CODE;
+        }
+        return guestbookAtRequest;
+    }
+
+    public void setGuestbookAtRequest(String gbAtRequest) {
+        if (gbAtRequest != null && gbAtRequest.equals(UNDEFINED_CODE)) {
+            this.guestbookAtRequest = null;
+        } else {
+            //Force to true or false
+            this.guestbookAtRequest = Boolean.valueOf(Boolean.parseBoolean(gbAtRequest)).toString();
+        }
     }
     
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index 31cea6d09cd..a4a0c1353b8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -6,6 +6,7 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.Setting;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean.Key;
@@ -23,6 +24,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.logging.Logger;
 import java.util.Set;
 
@@ -588,7 +590,7 @@ public Map<String, String> getBaseMetadataLanguageMap(boolean refresh) {
     public Map<String, String> getMetadataLanguages(DvObjectContainer target) {
         Map<String,String> currentMap = new HashMap<String,String>();
         currentMap.putAll(getBaseMetadataLanguageMap(false));
-        currentMap.put(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE, getDefaultMetadataLanguageLabel(target));
+        currentMap.put(DvObjectContainer.UNDEFINED_CODE, getDefaultMetadataLanguageLabel(target));
         return currentMap;
     }
 
@@ -599,7 +601,7 @@ private String getDefaultMetadataLanguageLabel(DvObjectContainer target) {
             String mlCode = target.getOwner().getEffectiveMetadataLanguage();
 
             // If it's undefined, no parent has a metadata language defined, and the global default should be used.
-            if (!mlCode.equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)) {
+            if (!mlCode.equals(DvObjectContainer.UNDEFINED_CODE)) {
                 // Get the label for the language code found
                 mlLabel = getBaseMetadataLanguageMap(false).get(mlCode);
                 mlLabel = mlLabel + " " + BundleUtil.getStringFromBundle("dataverse.inherited");
@@ -617,7 +619,7 @@ public String getDefaultMetadataLanguage() {
             return (String) mdMap.keySet().toArray()[0];
             } else {
                 //More than one - :MetadataLanguages is set and the default is undefined (users must choose if the collection doesn't override the default)
-                return DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE;
+                return DvObjectContainer.UNDEFINED_CODE;
             }
         } else {
             // None - :MetadataLanguages is not set so return null to turn off the display (backward compatibility)
@@ -625,6 +627,32 @@ public String getDefaultMetadataLanguage() {
         }
     }
     
+    public Map<String, String> getGuestbookEntryOptions(DvObjectContainer target) {
+        Map<String, String> currentMap = new HashMap<String, String>();
+        String atDownload = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atdownload");
+        String atRequest = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atrequest");
+        Optional<Boolean> gbDefault = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class);
+        if (gbDefault.isPresent()) {
+            // Three options - inherited/default option, at Download, at Request
+            String useDefault = null;
+            if (target.getOwner() == null) {
+                boolean defaultOption = gbDefault.get();
+                useDefault = (defaultOption ? atRequest : atDownload)
+                        + BundleUtil.getStringFromBundle("dataverse.default");
+            } else {
+                boolean defaultOption = target.getOwner().getEffectiveGuestbookEntryAtRequest();
+                useDefault = (defaultOption ? atRequest : atDownload)
+                        + BundleUtil.getStringFromBundle("dataverse.inherited");
+            }
+            currentMap.put("null", useDefault);
+            currentMap.put(Boolean.toString(true), atRequest);
+            currentMap.put(Boolean.toString(false), atDownload);
+        } else {
+            // Setting not defined - leave empty
+        }
+        return currentMap;
+    }
+    
     public Dataverse getRootDataverse() {
         if (rootDataverse == null) {
             rootDataverse = dataverseService.findRootDataverse();
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java
index 04e7d56fad6..76bfb3bf609 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataverse/DataverseUtil.java
@@ -87,7 +87,7 @@ public static void checkMetadataLangauge(Dataset ds, Dataverse owner, Map<String
         // :MetadataLanguage setting is not set
         // Must send UNDEFINED or match parent
         if (mLangMap.isEmpty()) {
-            if (!(ds.getMetadataLanguage().equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)
+            if (!(ds.getMetadataLanguage().equals(DvObjectContainer.UNDEFINED_CODE)
                     || ds.getMetadataLanguage().equals(owner.getMetadataLanguage()))) {
                 throw new BadRequestException("This repository is not configured to support metadataLanguage.");
             }
@@ -96,8 +96,8 @@ public static void checkMetadataLangauge(Dataset ds, Dataverse owner, Map<String
             // parent collection choice, or, if that is undefined, be one of the choices
             // allowed by the setting
             if (!((ds.getMetadataLanguage().equals(owner.getMetadataLanguage())
-                    && !owner.getMetadataLanguage().equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE))
-                    || (owner.getMetadataLanguage().equals(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE)
+                    && !owner.getMetadataLanguage().equals(DvObjectContainer.UNDEFINED_CODE))
+                    || (owner.getMetadataLanguage().equals(DvObjectContainer.UNDEFINED_CODE)
                             && (mLangMap.containsKey(ds.getMetadataLanguage()))))) {
                 throw new BadRequestException("Specified metadatalanguage ( metadataLanguage, "
                         + JsonLDTerm.schemaOrg("inLanguage").getUrl() + ") not allowed in this collection.");
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index c16a1f23bd1..938a628b544 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -918,6 +918,8 @@ dataverse.datasize.ioerror=Fatal IO error while trying to determine the total si
 dataverse.inherited=(inherited from enclosing Dataverse)
 dataverse.default=(Default)
 dataverse.metadatalanguage.setatdatasetcreation=Chosen at Dataset Creation
+dataverse.guestbookentry.atdownload=Guestbook Entry At Download
+dataverse.guestbookentry.atrequest=Guestbook Entry At Access Request
 # rolesAndPermissionsFragment.xhtml
 
 # advanced.xhtml
diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml
index aa3fa535807..b6bb8f70962 100644
--- a/src/main/webapp/dataverse.xhtml
+++ b/src/main/webapp/dataverse.xhtml
@@ -252,6 +252,8 @@
                                 </div>
                             </div>
                             <div class="col-md-6 form-group">
+                                <div class="row">
+                                    <div class="col-xs-12 form-group">
                                 <h:outputLabel for="description" styleClass="control-label">
                                     #{bundle.description}
                                     <span class="glyphicon glyphicon-question-sign tooltip-icon"
@@ -267,6 +269,21 @@
                                     </p:inputTextarea>
                                     <p:message for="description" display="text"/>
                                 </div>
+                                </div>
+                                    <div class="col-xs-12 form-group" jsf:rendered="#{DataversePage.getGuestbookEntryOptions().size()>2}">
+                                        <h:outputLabel for="dsGuestbookEntryOption" styleClass="control-label">
+                                            #{bundle.metadataLanguage} 
+                                            <span class="glyphicon glyphicon-question-sign tooltip-icon"
+                                                  data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.guestbookentryatrequest.title']}"></span>
+                                        </h:outputLabel>
+                                        <div class="form-col-container">
+                                            <h:selectOneMenu id="dsGuestbookEntryOption" styleClass="form-control" value="#{DataversePage.dataverse.guestbookEntryAtRequest}">
+                                                <f:selectItems value="#{DataversePage.getGuestbookEntryOptions()}" var="option" itemLabel="#{option.getValue()}" itemValue="#{option.getKey()}"/>
+                                            </h:selectOneMenu>
+                                            <p:message for="dsGuestbookEntryOptions" display="text"/>
+                                        </div>
+                                    </div>
+                                </div>
                             </div>
                         </div>
                         <hr class="margin-top-half margin-bottom-half"/>
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java
index bf679f8fe97..7053ef29412 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataverse/DataverseUtilTest.java
@@ -31,9 +31,9 @@ public void testCheckMetadataLanguageCases() {
         mLangSettingMap.put("en", "English");
         mLangSettingMap.put("fr", "French");
         Dataverse undefinedParent = new Dataverse();
-        undefinedParent.setMetadataLanguage(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE);
+        undefinedParent.setMetadataLanguage(DvObjectContainer.UNDEFINED_CODE);
         Dataset undefinedD = new Dataset();
-        undefinedD.setMetadataLanguage(DvObjectContainer.UNDEFINED_METADATA_LANGUAGE_CODE);
+        undefinedD.setMetadataLanguage(DvObjectContainer.UNDEFINED_CODE);
         Dataverse definedParent = new Dataverse();
         definedParent.setMetadataLanguage("en");
         Dataset definedEnglishD = new Dataset();

From 2d1c5c5f5592019f84f8961689aec708c4f6b1da Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 14:19:35 -0400
Subject: [PATCH 218/922] update flyway to add column guestbookatrequest

---
 .../resources/db/migration/V5.13.0.3__guestbook-on-request.sql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
index a92d8910f6d..7e05e792858 100644
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
@@ -7,3 +7,6 @@ ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_g
 ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);
 DROP INDEX IF EXISTS created_requests;
 CREATE UNIQUE INDEX created_requests ON fileaccessrequests (datafile_id, authenticated_user_id) WHERE request_state='CREATED';
+
+ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS guestbookatrequest bool;
+ALTER TABLE dataset ADD COLUMN IF NOT EXISTS guestbookatrequest bool;

From 39fefa09b28b1b472ffdfb3454376d33dfd455ce Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 14:19:56 -0400
Subject: [PATCH 219/922] change setting to match standard format

---
 .../java/edu/harvard/iq/dataverse/settings/JvmSettings.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index 860ae6e8fab..41ac030bd0a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -49,7 +49,7 @@ public enum JvmSettings {
     // FILES SETTINGS
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
-    GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbookAtRequest"),
+    GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),

From 1c201ad311a64574bccf2a123d69a251a273292e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 14:20:19 -0400
Subject: [PATCH 220/922] minor fixes for collection-level option

---
 src/main/java/edu/harvard/iq/dataverse/DataversePage.java     | 3 +++
 src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java | 2 +-
 src/main/java/propertyFiles/Bundle.properties                 | 1 +
 src/main/webapp/dataverse.xhtml                               | 4 ++--
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
index b48ff725e1e..257cbc17033 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
@@ -1287,4 +1287,7 @@ public String getCurationLabelSetNameLabel() {
         return setName;
     }
 
+    public Set<Entry<String, String>> getGuestbookEntryOptions() {
+        return settingsWrapper.getGuestbookEntryOptions(this.dataverse).entrySet();
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index 7d8a741b267..713c5bc4069 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -121,7 +121,7 @@ public String getGuestbookEntryAtRequest() {
         return guestbookAtRequest;
     }
 
-    public void setGuestbookAtRequest(String gbAtRequest) {
+    public void setGuestbookEntryAtRequest(String gbAtRequest) {
         if (gbAtRequest != null && gbAtRequest.equals(UNDEFINED_CODE)) {
             this.guestbookAtRequest = null;
         } else {
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 84661e5cc17..a96f9266219 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -54,6 +54,7 @@ affiliation=Affiliation
 storage=Storage
 curationLabels=Curation Labels
 metadataLanguage=Dataset Metadata Language
+guestbookEntryOption=Guestbook Entry Option
 createDataverse=Create Dataverse
 remove=Remove
 done=Done
diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml
index b6bb8f70962..e4484d38b97 100644
--- a/src/main/webapp/dataverse.xhtml
+++ b/src/main/webapp/dataverse.xhtml
@@ -272,7 +272,7 @@
                                 </div>
                                     <div class="col-xs-12 form-group" jsf:rendered="#{DataversePage.getGuestbookEntryOptions().size()>2}">
                                         <h:outputLabel for="dsGuestbookEntryOption" styleClass="control-label">
-                                            #{bundle.metadataLanguage} 
+                                            #{bundle.guestbookEntryOption} 
                                             <span class="glyphicon glyphicon-question-sign tooltip-icon"
                                                   data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.guestbookentryatrequest.title']}"></span>
                                         </h:outputLabel>
@@ -280,7 +280,7 @@
                                             <h:selectOneMenu id="dsGuestbookEntryOption" styleClass="form-control" value="#{DataversePage.dataverse.guestbookEntryAtRequest}">
                                                 <f:selectItems value="#{DataversePage.getGuestbookEntryOptions()}" var="option" itemLabel="#{option.getValue()}" itemValue="#{option.getKey()}"/>
                                             </h:selectOneMenu>
-                                            <p:message for="dsGuestbookEntryOptions" display="text"/>
+                                            <p:message for="dsGuestbookEntryOption" display="text"/>
                                         </div>
                                     </div>
                                 </div>

From 24490fa3a3bfd44890dbe91348f36d702cd154db Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 14:49:14 -0400
Subject: [PATCH 221/922] update check to use effective value

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index dac3f6409e3..e087aa1232c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5264,7 +5264,7 @@ public boolean isGuestbookPopupRequiredAtDownload(){
         // Only show guestbookAtDownload if fileAccessRequest is turned off for the
         // dataset or guestbook at request is disabled (legacy behavior)
         return isGuestbookPopupRequired() && (!workingVersion.getDataset().isFileAccessRequest()
-                || !(JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class).orElse(false)));
+                || !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest());
     }
     
     public String requestAccessMultipleFiles() {

From 5db7a65f456a33e5d9668a23967030aca63e799e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 15:33:47 -0400
Subject: [PATCH 222/922] add api

---
 .../harvard/iq/dataverse/api/Datasets.java    | 80 +++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index dcd7eacf50b..643c701ae80 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3879,4 +3879,84 @@ public Response getDatasetVersionCitation(@Context ContainerRequestContext crc,
         return response(req -> ok(
                 getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getCitation(true, false)), getRequestUser(crc));
     }
+    
+    @GET
+    @AuthRequired
+    @Path("{identifier}/guestbookEntryAtRequest")
+    public Response getGuestbookEntryOption(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf,
+            @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse { 
+        
+        Dataset dataset; 
+        
+        try {
+            dataset = findDatasetOrDie(dvIdtf);
+        } catch (WrappedResponse ex) {
+            return error(Response.Status.NOT_FOUND, "No such dataset");
+        }
+        
+        return response(req -> ok(dataset.getEffectiveGuestbookEntryAtRequest()), getRequestUser(crc));
+    }
+    
+    @PUT
+    @AuthRequired
+    @Path("{identifier}/guestbookEntryAtRequest")
+    public Response setguestbookEntryAtRequest(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf,
+            boolean gbAtRequest,
+            @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse {
+        
+        // Superuser-only:
+        AuthenticatedUser user;
+        try {
+            user = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse ex) {
+            return error(Response.Status.BAD_REQUEST, "Authentication is required.");
+        }
+        if (!user.isSuperuser()) {
+            return error(Response.Status.FORBIDDEN, "Superusers only.");
+        }
+
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(dvIdtf);
+        } catch (WrappedResponse ex) {
+            return error(Response.Status.NOT_FOUND, "No such dataset");
+        }
+        
+        String choice = Boolean.valueOf(gbAtRequest).toString();
+        dataset.setGuestbookEntryAtRequest(choice);
+        datasetService.merge(dataset);
+        return ok("Guestbook Entry At Request set to: " + choice);
+    }
+    
+    @DELETE
+    @AuthRequired
+    @Path("{identifier}/guestbookEntryAtRequest")
+    public Response resetGuestbookEntryAtRequest(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf,
+            @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse {
+        
+        // Superuser-only:
+        AuthenticatedUser user;
+        try {
+            user = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse ex) {
+            return error(Response.Status.BAD_REQUEST, "Authentication is required.");
+        }
+        if (!user.isSuperuser()) {
+            return error(Response.Status.FORBIDDEN, "Superusers only.");
+        }
+
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(dvIdtf);
+        } catch (WrappedResponse ex) {
+            return error(Response.Status.NOT_FOUND, "No such dataset");
+        }
+        
+        dataset.setGuestbookEntryAtRequest(DvObjectContainer.UNDEFINED_CODE);
+        datasetService.merge(dataset);
+        return ok("Guestbook Entry At Request reset to default: " + dataset.getEffectiveGuestbookEntryAtRequest());
+    }
+
 }

From dc89f8f05482954400461dcafb40311d1e9d5c16 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 15:34:19 -0400
Subject: [PATCH 223/922] add setting response when dataset has a value

---
 src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index 713c5bc4069..2b3b49d8dfd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -110,6 +110,8 @@ public boolean getEffectiveGuestbookEntryAtRequest() {
                 gbAtRequest = opt.get();
                 }
             }
+        } else {
+            gbAtRequest = Boolean.parseBoolean(guestbookAtRequest);
         }
         return gbAtRequest;
     }

From 44056054d26615031fa891a2e3c4c2417328bb96 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 15:48:14 -0400
Subject: [PATCH 224/922] Don't allow setting this when the config setting is
 not set.

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 643c701ae80..11d7d83ef2a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3922,7 +3922,10 @@ public Response setguestbookEntryAtRequest(@Context ContainerRequestContext crc,
         } catch (WrappedResponse ex) {
             return error(Response.Status.NOT_FOUND, "No such dataset");
         }
-        
+        Optional<Boolean> gbAtRequestOpt = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class);
+        if (!gbAtRequestOpt.isPresent()) {
+            return error(Response.Status.FORBIDDEN, "Guestbook Entry At Request cannot be set. This server is not configured to allow it.");
+        }
         String choice = Boolean.valueOf(gbAtRequest).toString();
         dataset.setGuestbookEntryAtRequest(choice);
         datasetService.merge(dataset);

From 5ce3eb35015342966005ccd63becce96a9d91608 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 16:44:40 -0400
Subject: [PATCH 225/922] report default/inherited setting as well.

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 11d7d83ef2a..e84f0108a3c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3893,8 +3893,11 @@ public Response getGuestbookEntryOption(@Context ContainerRequestContext crc, @P
         } catch (WrappedResponse ex) {
             return error(Response.Status.NOT_FOUND, "No such dataset");
         }
-        
-        return response(req -> ok(dataset.getEffectiveGuestbookEntryAtRequest()), getRequestUser(crc));
+        String gbAtRequest = dataset.getGuestbookEntryAtRequest();
+        if(gbAtRequest == null || gbAtRequest.equals(DvObjectContainer.UNDEFINED_CODE)) {
+            return ok("Not set on dataset, using the default: " + dataset.getEffectiveGuestbookEntryAtRequest());
+        }
+        return ok(dataset.getEffectiveGuestbookEntryAtRequest());
     }
     
     @PUT

From 3cf323c2eaee66995d13e699e7e8a1b67f977ee6 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 16:44:53 -0400
Subject: [PATCH 226/922] Missing help text

---
 src/main/java/propertyFiles/Bundle.properties | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index a96f9266219..4873ed3c52d 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -822,6 +822,7 @@ dataverse.curationLabels.title=A set of curation status labels that are used to
 dataverse.curationLabels.disabled=Disabled
 dataverse.category=Category
 dataverse.category.title=The type that most closely reflects this dataverse.
+dataverse.guestbookentryatrequest.title=Whether Guestbooks are displayed to users when they request file access or when they download files.
 dataverse.type.selectTab.top=Select one...
 dataverse.type.selectTab.researchers=Researcher
 dataverse.type.selectTab.researchProjects=Research Project

From 8b91f2c599f6b57ed9f7409ff581c174b6599419 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 12 Jul 2023 16:45:07 -0400
Subject: [PATCH 227/922] documentation/release note

---
 .../9599-guestbook-at-request.md              |  2 +
 doc/sphinx-guides/source/api/native-api.rst   | 45 +++++++++++++++++++
 .../source/installation/config.rst            | 11 +++++
 .../source/user/dataverse-management.rst      |  2 +
 4 files changed, 60 insertions(+)
 create mode 100644 doc/release-notes/9599-guestbook-at-request.md

diff --git a/doc/release-notes/9599-guestbook-at-request.md b/doc/release-notes/9599-guestbook-at-request.md
new file mode 100644
index 00000000000..72255150f37
--- /dev/null
+++ b/doc/release-notes/9599-guestbook-at-request.md
@@ -0,0 +1,2 @@
+Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
+The global default defined by this setting can be overridden at the collection level on the collection page and at the inidividual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions. 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index ccf172eb84e..e17f16e3f54 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2237,6 +2237,51 @@ See :ref:`:CustomDatasetSummaryFields` in the Installation Guide for how the lis
 
   curl "$SERVER_URL/api/datasets/summaryFieldNames"
 
+.. _guestbook-at-request-api:
+  
+Configure When a Dataset Guestbook Appears (If Enabled)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, users are asked to fill out a configured Guestbook when they down download files from a dataset. If enabled for a given Dataverse instance (see XYZ), users may instead be asked to fill out a Guestbook only when they request access to restricted files.
+This is configured by a global default, collection-level settings, or directly at the dataset level via these API calls (superuser access is required to make changes).
+
+To see the current choice for this dataset:
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+
+  curl "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+  
+  
+  The response will be true (guestbook displays when making a request), false (guestbook displays at download), or will indicate that the dataset inherits one of these settings.
+
+To set the behavior for this dataset:
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+
+  curl -X PUT "-H:X-Dataverse-key:$API_TOKEN" -H Content-type:application/json -d true "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+
+
+  This example uses true to set the behavior to guestbook at request. Note that this call will return a 403/Forbidden response if guestbook at request functionality is not enabled for this Dataverse instance.
+  
+The API can also be used to reset the dataset to use the default/inherited value:
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+
+  curl -X DELETE "-H:X-Dataverse-key:$API_TOKEN" -H Content-type:application/json "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+
+
+
 Files
 -----
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index ca17fe09c15..f332ca130ef 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -2442,6 +2442,17 @@ This setting was added to keep S3 direct upload lightweight. When that feature i
 
 See also :ref:`s3-direct-upload-features-disabled`.
 
+.. _dataverse.files.guestbook-at-request:
+
+dataverse.files.guestbook-at-request
+++++++++++++++++++++++++++++++++++++
+
+This setting enables functionality to allow guestbooks to be displayed when a user requests access to a restricted data file(s) or when a file is downloaded (the historic default). Providing a true/false value for this setting enables the functionality and provides a global default. The behavior can also be changed at the collection level via the user interface and by a superuser for a give dataset using the API.
+
+See also :ref:`guestbook-at-request-api` in the API Guide, and .
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_GUESTBOOK_AT_REQUEST``.
+
 .. _feature-flags:
 
 Feature Flags
diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst
index b5e8d8f4fc9..3d301619105 100755
--- a/doc/sphinx-guides/source/user/dataverse-management.rst
+++ b/doc/sphinx-guides/source/user/dataverse-management.rst
@@ -25,6 +25,8 @@ Creating a Dataverse collection is easy but first you must be a registered user
     * **Category**: Select a category that best describes the type of Dataverse collection this will be. For example, if this is a Dataverse collection for an individual researcher's datasets, select *Researcher*. If this is a Dataverse collection for an institution, select *Organization or Institution*.
     * **Email**: This is the email address that will be used as the contact for this particular Dataverse collection. You can have more than one contact email address for your Dataverse collection.
     * **Description**: Provide a description of this Dataverse collection. This will display on the landing page of your Dataverse collection and in the search result list. The description field supports certain HTML tags, if you'd like to format your text (<a>, <b>, <blockquote>, <br>, <code>, <del>, <dd>, <dl>, <dt>, <em>, <hr>, <h1>-<h3>, <i>, <img>, <kbd>, <li>, <ol>, <p>, <pre>, <s>, <sup>, <sub>, <strong>, <strike>, <u>, <ul>).
+    * **Dataset Metadata Langauge**: (If enabled) Select which language should be used when entering dataset metadata, or leave that choice to dataset creators. 
+    * **Guestbook Entry Option**: (If enabled) Select whether guestbooks are displayed when a user requests access to restricted file(s) or when they initiate a download.
 #. **Choose the sets of Metadata Fields for datasets in this Dataverse collection**:
     * By default the metadata elements will be from the host Dataverse collection that this new Dataverse collection is created in.
     * The Dataverse Software offers metadata standards for multiple domains. To learn more about the metadata standards in the Dataverse Software please check out the :doc:`/user/appendix`.

From 489b990d63ab0a2ac4af02dd7725148134107a76 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 17 Jul 2023 13:59:03 +0100
Subject: [PATCH 228/922] Added: dataTables field to DataFile JSON API payload

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 61134ae5245..38fedc70fc8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -693,7 +693,8 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
-                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
+                .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables()));
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the

From 8c2781b1096a457ee824be57623dc0bc113da316 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 17 Jul 2023 14:09:12 +0100
Subject: [PATCH 229/922] Added: release notes for 9692

---
 doc/release-notes/9692-files-api-extension.md | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 doc/release-notes/9692-files-api-extension.md

diff --git a/doc/release-notes/9692-files-api-extension.md b/doc/release-notes/9692-files-api-extension.md
new file mode 100644
index 00000000000..420da17d9b6
--- /dev/null
+++ b/doc/release-notes/9692-files-api-extension.md
@@ -0,0 +1,7 @@
+The following API endpoints have been added:
+
+- /api/files/{id}/guestbookResponses/count
+- /api/files/{id}/canBeDownloaded
+- /api/files/{id}/thumbnailClass
+
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination and ordering

From 91e5a8e924023ea1fe6861215fa9d83a95282bab Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 17 Jul 2023 10:29:17 -0400
Subject: [PATCH 230/922] Start making params indep of hostPage (dataset or
 file)

---
 .../edu/harvard/iq/dataverse/FilePage.java    | 22 +++++++++++++++++++
 src/main/webapp/dataset.xhtml                 |  2 ++
 src/main/webapp/file.xhtml                    |  2 ++
 .../guestbook-terms-popup-fragment.xhtml      | 10 ++++-----
 4 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
index 70faeeac815..fa8b494bce2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
@@ -312,6 +312,20 @@ private List<ExternalTool> sortExternalTools(){
         Collections.sort(retList, CompareExternalToolName);
         return retList;
     }
+    
+    private String termsGuestbookPopupAction = "";
+
+    public void setTermsGuestbookPopupAction(String popupAction){
+        if(popupAction != null && popupAction.length() > 0){
+            logger.info("TGPA set to " + popupAction);
+            this.termsGuestbookPopupAction = popupAction;
+        }
+
+    }
+
+    public String getTermsGuestbookPopupAction(){
+        return termsGuestbookPopupAction;
+    }
 
     public boolean isDownloadPopupRequired() {  
         if(fileMetadata.getId() == null || fileMetadata.getDatasetVersion().getId() == null ){
@@ -333,6 +347,14 @@ public boolean isGuestbookAndTermsPopupRequired() {
         }
         return FileUtil.isGuestbookAndTermsPopupRequired(fileMetadata.getDatasetVersion());
     }
+    
+    public boolean isGuestbookPopupRequiredAtDownload(){
+        // Only show guestbookAtDownload if fileAccessRequest is turned off for the
+        // dataset or guestbook at request is disabled (legacy behavior)
+        DatasetVersion workingVersion = fileMetadata.getDatasetVersion();
+        return FileUtil.isGuestbookPopupRequired(workingVersion) && (!workingVersion.getDataset().isFileAccessRequest()
+                || !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest());
+    }
 
     public void setFileMetadata(FileMetadata fileMetadata) {
         this.fileMetadata = fileMetadata;
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 1cc8213ecd5..c9e289dac00 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1517,6 +1517,8 @@
                             <ui:param name="guestbookResponse" value="#{DatasetPage.guestbookResponse}" rendered="#{!settingsWrapper.rsyncDownload}"/>
                             <ui:param name="fileDownloadHelper" value="#{DatasetPage.fileDownloadHelper}"/>
                             <ui:param name="fileDownloadService" value="#{DatasetPage.fileDownloadService}"/>
+                            <ui:param name="termsGuestbookPopupAction" value="#{DatasetPage.termsGuestbookPopupAction}"/>
+                            <ui:param name="guestbookPopupRequiredAtDownload" value="#{DatasetPage.guestbookPopupRequiredAtDownload}"/>
                         </ui:include>
                     </p:dialog>
                     <!-- Preview Guestbook -->
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index 05bd345c011..bda8e50de00 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -371,6 +371,8 @@
                                         <ui:param name="guestbookResponseService" value="#{FilePage.guestbookResponseService}"/>
                                         <ui:param name="fileDownloadService" value="#{FilePage.fileDownloadService}"/>
                                         <ui:param name="lockedFromDownload" value="#{FilePage.lockedFromDownload}"/>
+                                        <ui:param name="termsGuestbookPopupAction" value="#{FilePage.termsGuestbookPopupAction}"/>
+                                        <ui:param name="guestbookPopupRequiredAtDownload" value="#{FilePage.guestbookPopupRequiredAtDownload}"/>
                                     </ui:include>
                                 </ui:fragment>
                                 <!-- PREVIEW EXTERNAL TOOL -->
diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 4099d6faa1f..5e7276a82bc 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -10,7 +10,7 @@
 
         <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
         <p:focus context="guestbookUIFragment"/>
-        <ui:fragment rendered="#{DatasetPage.termsGuestbookPopupAction != bundle.download}">
+        <ui:fragment rendered="#{termsGuestbookPopupAction != bundle.download}">
             <p class="help-block">
                 #{someActivelyEmbargoedFiles ? bundle['file.requestAccessTermsDialog.embargoed.tip'] : bundle['file.requestAccessTermsDialog.tip']}
             </p>
@@ -18,7 +18,7 @@
                 #{bundle['file.requestAccessTermsDialog.embargoed']}
             </p>
         </ui:fragment>
-        <ui:fragment rendered="#{DatasetPage.termsGuestbookPopupAction != bundle.download}">
+        <ui:fragment rendered="#{termsGuestbookPopupAction != bundle.download}">
             <p class="help-block">
                 #{bundle['file.downloadDialog.tip']}
             </p>
@@ -141,7 +141,7 @@
                 value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}"
                 escape="false" />
         </div>
-            <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((DatasetPage.termsGuestbookPopupAction != bundle.download) != DatasetPage.guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
+            <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((termsGuestbookPopupAction != bundle.download) != guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
                 <!--
                 Only validate the active context. For example, the user might be filling out
                 the guestbook on the Preview tab rather than after clicking the "download file"
@@ -247,7 +247,7 @@
                 </div>
             </p:fragment>
         </div>
-        <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle.download}">
+        <div class="button-block" jsf:rendered="#{termsGuestbookPopupAction == bundle.download}">
         <!--
             The "process" directive below is very important. Without it, the
             setters on the GuestbookResponse object can be called twice leading
@@ -300,7 +300,7 @@
                 #{bundle.cancel}
             </button>
         </div>
-        <div class="button-block" jsf:rendered="#{DatasetPage.termsGuestbookPopupAction == bundle['file.requestAccess']}">
+        <div class="button-block" jsf:rendered="#{termsGuestbookPopupAction == bundle['file.requestAccess']}">
         <!--TODO - add "Accept and" if ToA exist?-->
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"                          

From 867fb8adb48ab0c402ede1e472f5d014d6485b02 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 18 Jul 2023 14:42:57 +0100
Subject: [PATCH 231/922] Added: endpoint for getting file data tables and
 missing authentication added to other endpoints

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 43 ++++++++++-------
 .../iq/dataverse/util/json/JsonPrinter.java   |  3 +-
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 46 +++++++++++++++++--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 +++
 src/test/resources/tab/test.tab               | 16 +++++++
 5 files changed, 93 insertions(+), 21 deletions(-)
 create mode 100644 src/test/resources/tab/test.tab

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 171f8aa6e87..f42f6f5abc7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -76,6 +76,8 @@
 import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonDT;
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import javax.ws.rs.core.UriInfo;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -829,15 +831,13 @@ public Response getFixityAlgorithm() {
     }
 
     @GET
+    @AuthRequired
     @Path("{id}/guestbookResponses/count")
-    public Response getCountGuestbookResponses(@PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
+    public Response getCountGuestbookResponses(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
+        }, getRequestUser(crc));
     }
 
     @GET
@@ -854,14 +854,25 @@ public Response canFileBeDownloaded(@Context ContainerRequestContext crc, @PathP
     }
 
     @GET
+    @AuthRequired
     @Path("{id}/thumbnailClass")
-    public Response getFileThumbnailClass(@PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        return ok(dataFileServiceBean.getFileThumbnailClass(dataFile));
+    public Response getFileThumbnailClass(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            return ok(dataFileServiceBean.getFileThumbnailClass(dataFile));
+        }, getRequestUser(crc));
+    }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/dataTables")
+    public Response getFileDataTables(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            if (!dataFile.isTabularData()) {
+                return error(BAD_REQUEST, "This operation is only available for tabular files.");
+            }
+            return ok(jsonDT(dataFile.getDataTables()));
+        }, getRequestUser(crc));
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 38fedc70fc8..61134ae5245 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -693,8 +693,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
-                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
-                .add("dataTables", df.getDataTables().isEmpty() ? null : JsonPrinter.jsonDT(df.getDataTables()));
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index a2dafffea1b..67502d632fe 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2107,7 +2107,7 @@ public void testGetCountGuestbookResponses() throws InterruptedException {
         Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
         // Upload test file
-        String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
         Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
         uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
@@ -2150,7 +2150,7 @@ public void testCanFileBeDownloaded() {
         int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
         // Upload test file
-        String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
         Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
         uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
@@ -2182,7 +2182,7 @@ public void testGetFileThumbnailClass() {
         int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
         // Upload test file
-        String pathToTestFile = "src/main/webapp/resources/images/dataverseproject.png";
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
         Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
         uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
@@ -2198,4 +2198,44 @@ public void testGetFileThumbnailClass() {
         Response getFileThumbnailClassInvalidIdResponse = UtilIT.getFileThumbnailClass("testInvalidId", apiToken);
         getFileThumbnailClassInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
+
+    @Test
+    public void testGetFileDataTables() throws InterruptedException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload non-tabular file
+        String pathToNonTabularTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadNonTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToNonTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadNonTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert that getting data tables for non-tabular file fails
+        int testNonTabularFileId = JsonPath.from(uploadNonTabularFileResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response getFileDataTablesForNonTabularFileResponse = UtilIT.getFileDataTables(Integer.toString(testNonTabularFileId), apiToken);
+        getFileDataTablesForNonTabularFileResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+
+        // Upload tabular file
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Ensure tabular file is ingested
+        sleep(2000);
+
+        // Get file data tables for the tabular file and assert data is obtained
+        int testTabularFileId = JsonPath.from(uploadTabularFileResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(Integer.toString(testTabularFileId), apiToken);
+        getFileDataTablesForTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+        int dataTablesNumber = JsonPath.from(getFileDataTablesForTabularFileResponse.body().asString()).getList("data").size();
+        assertTrue(dataTablesNumber > 0);
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 831543ca484..de3df705714 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3295,4 +3295,10 @@ static Response getFileThumbnailClass(String dataFileId, String apiToken) {
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/thumbnailClass");
     }
+
+    static Response getFileDataTables(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/files/" + dataFileId + "/dataTables");
+    }
 }
diff --git a/src/test/resources/tab/test.tab b/src/test/resources/tab/test.tab
new file mode 100644
index 00000000000..628a309a530
--- /dev/null
+++ b/src/test/resources/tab/test.tab
@@ -0,0 +1,16 @@
+test1 test2 test3 test4
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"
+"test1 test2 test3 test4"

From f2b374eaac45c13b34a545055a48bdad58605560 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 18 Jul 2023 14:45:53 +0100
Subject: [PATCH 232/922] Added: new endpoint to the release notes

---
 doc/release-notes/9692-files-api-extension.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/release-notes/9692-files-api-extension.md b/doc/release-notes/9692-files-api-extension.md
index 420da17d9b6..03484083113 100644
--- a/doc/release-notes/9692-files-api-extension.md
+++ b/doc/release-notes/9692-files-api-extension.md
@@ -3,5 +3,6 @@ The following API endpoints have been added:
 - /api/files/{id}/guestbookResponses/count
 - /api/files/{id}/canBeDownloaded
 - /api/files/{id}/thumbnailClass
+- /api/files/{id}/dataTables
 
 The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination and ordering

From 033da2381bf86451e3897eb0413b10b406271a67 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 10:28:18 -0400
Subject: [PATCH 233/922] use bean param for DatasetPage/FilePage

---
 .../file-download-button-fragment.xhtml       | 20 +++++++++----------
 src/main/webapp/file.xhtml                    |  4 ++++
 src/main/webapp/filesFragment.xhtml           |  1 +
 3 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index 5c29cd6b041..32cd837f3b1 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -31,7 +31,7 @@
                         update="@([id$=requestAccessConsolidated]), @([id$=requestPanel]),  @([id$=filesTable]), @([id$=messagePanel]), @([id$=guestbookAndTermsPopup])"
                         id="requestAccessConsolidated"
                         disabled="#{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user)}">
-                 <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle['file.requestAccess'])}"/>
+                 <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle['file.requestAccess'])}"/>
                 #{fileMetadata.dataFile.containsActiveFileAccessRequestFromUser(dataverseSession.user) ? bundle['file.accessRequested'] : bundle['file.requestAccess']}
             </p:commandLink>
         </p:outputPanel>
@@ -77,7 +77,7 @@
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                          action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'GlobusTransfer')}"
                          update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                         <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                         <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
                 GT: #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
@@ -105,7 +105,7 @@
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                          action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'package')}"
                          update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                         <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                         <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <f:actionListener binding="#{packagePopupFragmentBean.setFileMetadata(fileMetadata)}" /> 
                 <!-- package data file: -->
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
@@ -129,7 +129,7 @@
                          action="#{guestbookResponseService.modifyDatafile(guestbookResponse, fileMetadata)}"
                          update="@widgetVar(guestbookAndTermsPopup)" oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
                 <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
-                <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
@@ -148,7 +148,7 @@
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'bundle' )}"
                                update="@widgetVar(guestbookAndTermsPopup)"
                                oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                               <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                     #{bundle['file.downloadBtn.format.all']}
                 </p:commandLink>
             </li>
@@ -169,7 +169,7 @@
                                action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'original' )}"
                                update="@widgetVar(guestbookAndTermsPopup)"
                                oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                    <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                    <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                     <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                     <h:outputFormat value="#{bundle['file.downloadBtn.format.original']}">
                         <f:param value="#{fileMetadata.dataFile.originalFormatLabel}"/>
@@ -188,7 +188,7 @@
                                disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
                                update="@widgetVar(guestbookAndTermsPopup)"
                                oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                    <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                    <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                     <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                     #{bundle['file.downloadBtn.format.tab']}
                 </p:commandLink>
@@ -207,7 +207,7 @@
                                    action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'RData' )}"
                                    update="@widgetVar(guestbookAndTermsPopup)"
                                    oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                        <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                        <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                         <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                         #{bundle['file.downloadBtn.format.rdata']}
                     </p:commandLink>
@@ -235,7 +235,7 @@
                            action="#{guestbookResponseService.modifyDatafileAndFormat(guestbookResponse, fileMetadata, 'var' )}"
                            update="@widgetVar(guestbookAndTermsPopup)"
                            oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                           <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                           <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                 #{bundle['file.downloadBtn.format.var']}
             </p:commandLink>
         </li>
@@ -326,7 +326,7 @@
                                process="@this"
                                update="@widgetVar(guestbookAndTermsPopup)"
                                oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
-                               <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                               <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                     #{tool.getDisplayNameLang()}
                 </p:commandLink>
             </li>
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index bda8e50de00..fa13c5fbe40 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -206,6 +206,7 @@
                                                                 <ui:param name="lockedFromDownload" value="#{FilePage.lockedFromDownload}"/>
                                                                 <ui:param name="exploreTools" value="#{FilePage.exploreTools}"/>
                                                                 <ui:param name="anonymized" value="#{FilePage.anonymizedAccess}"/>
+                                                                <ui:param name="bean" value="#{FilePage}"/>
                                                             </ui:include>
                                                         </ui:fragment>
                                                         <!-- END: Explore/Download/Request Button Block -->
@@ -656,6 +657,9 @@
                             <ui:param name="guestbookResponse" value="#{FilePage.guestbookResponse}"/>
                             <ui:param name="fileDownloadHelper" value="#{FilePage.fileDownloadHelper}"/>
                             <ui:param name="fileDownloadService" value="#{FilePage.fileDownloadService}"/>
+                            <ui:param name="termsGuestbookPopupAction" value="#{FilePage.termsGuestbookPopupAction}"/>
+                            <ui:param name="guestbookPopupRequiredAtDownload" value="#{FilePage.guestbookPopupRequiredAtDownload}"/>
+                            
                         </ui:include>
                     </p:dialog>
                     <p:dialog id="mapDataDialog" header="#{bundle['file.mapData.unpublished.header']}" widgetVar="mapDataDialog" modal="true">
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 6e1b2c30ded..c68cbd30a35 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -556,6 +556,7 @@
                             <ui:param name="lockedFromDownload" value="#{DatasetPage.lockedFromDownload}"/>
                             <ui:param name="exploreTools" value="#{DatasetPage.getExploreToolsForDataFile(fileMetadata.dataFile.id)}"/>
                             <ui:param name="anonymized" value="#{DatasetPage.anonymizedAccess}"/>
+                            <ui:param name="bean" value="#{DatasetPage}"/>
                         </ui:include>
                         <!-- END: Explore/Download/Request Button Block -->
 

From 0d29e439ed343baee74cdae7be68745eee02573c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 10:28:58 -0400
Subject: [PATCH 234/922] remove unused legacy worldmap-related dialog

---
 src/main/webapp/file.xhtml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index fa13c5fbe40..b4b52e60752 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -662,16 +662,6 @@
                             
                         </ui:include>
                     </p:dialog>
-                    <p:dialog id="mapDataDialog" header="#{bundle['file.mapData.unpublished.header']}" widgetVar="mapDataDialog" modal="true">
-                        <p class="help-block">
-                            <span class="text-danger"><span class="glyphicon glyphicon-exclamation-sign"/> #{bundle['file.mapData.unpublished.message']}</span>
-                        </p>
-                        <div class="button-block">
-                            <button type="button" class="btn btn-default" onclick="PF('mapDataDialog').hide();PF('blockDatasetForm').hide();">
-                                #{bundle.close}
-                            </button>
-                        </div>
-                    </p:dialog>                    
                     <ui:include rendered="#{systemConfig.provCollectionEnabled}" src="provenance-popups-fragment.xhtml">
                         <ui:param name="saveInPopup" value="true"/>
                     </ui:include>

From f6caac7d71d06a5517d20fe6aaed8311bb6a575c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 13:08:24 -0400
Subject: [PATCH 235/922] Backward compat - only report open requests, fix err
 response code

---
 src/main/java/edu/harvard/iq/dataverse/api/Access.java   | 4 ++--
 src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 70b1bdcdd2f..b645160d37c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1479,11 +1479,11 @@ public Response listFileAccessRequests(@Context ContainerRequestContext crc, @Pa
             return error(FORBIDDEN, BundleUtil.getStringFromBundle("access.api.rejectAccess.failure.noPermissions"));
         }
 
-        List<FileAccessRequest> requests = dataFile.getFileAccessRequests();
+        List<FileAccessRequest> requests = dataFile.getFileAccessRequests(FileAccessRequest.RequestState.CREATED);
 
         if (requests == null || requests.isEmpty()) {
             List<String> args = Arrays.asList(dataFile.getDisplayName());
-            return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestList.noRequestsFound", args));
+            return error(Response.Status.NOT_FOUND, BundleUtil.getStringFromBundle("access.api.requestList.noRequestsFound", args));
         }
 
         JsonArrayBuilder userArray = Json.createArrayBuilder();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index d6aac80b435..b6fb9afbc57 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -548,7 +548,7 @@ public void testRequestAccess() throws InterruptedException {
         assertEquals(200, revokeFileAccessResponse.getStatusCode());
 
         listAccessRequestResponse = UtilIT.getAccessRequestList(tabFile3IdRestrictedNew.toString(), apiToken);
-        assertEquals(400, listAccessRequestResponse.getStatusCode());
+        assertEquals(404, listAccessRequestResponse.getStatusCode());
     }
 
     // This is a round trip test of uploading a zipped archive, with some folder

From deb5a9fe723dd6996aadc1b6696571de8bb56260 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 14:15:26 -0400
Subject: [PATCH 236/922] don't submit empty gbr on access with gb at download

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 5e7276a82bc..d8f018054d6 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -302,15 +302,15 @@
         </div>
         <div class="button-block" jsf:rendered="#{termsGuestbookPopupAction == bundle['file.requestAccess']}">
         <!--TODO - add "Accept and" if ToA exist?-->
-            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
-                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"                          
+            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{!guestbookPopupRequiredAtDownload}" value="#{bundle['file.requestAccess']}"
+                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"
                     update="@([id$=Messages])"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
                     <f:param name="DO_GB_VALIDATION" value="true"/>
             </p:commandLink>
 
-            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{!workingVersion.dataset.hasEnabledGuestbook()}" value="#{bundle['file.requestAccess']}"
-                    actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"
+            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{guestbookPopupRequiredAtDownload}" value="bob #{bundle['file.requestAccess']}"
+                    actionListener="#{fileDownloadHelper.requestAccessIndirect()}"
                     update="@([id$=Messages])"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
                     <f:param name="DO_GB_VALIDATION" value="true"/>

From 5c79e55c2d879c51f023886e68514ffd2880fed0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 15:03:29 -0400
Subject: [PATCH 237/922] remove debug

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index d8f018054d6..55e11c9c386 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -309,7 +309,7 @@
                     <f:param name="DO_GB_VALIDATION" value="true"/>
             </p:commandLink>
 
-            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{guestbookPopupRequiredAtDownload}" value="bob #{bundle['file.requestAccess']}"
+            <p:commandLink type="button" styleClass="btn btn-default" rendered="#{guestbookPopupRequiredAtDownload}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.requestAccessIndirect()}"
                     update="@([id$=Messages])"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">

From a212aac2c8088b917847d862430a82ac32d94bfb Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 15:03:51 -0400
Subject: [PATCH 238/922] enhance request access email to owner

---
 .../harvard/iq/dataverse/MailServiceBean.java | 11 ++++++++
 .../iq/dataverse/util/json/JsonPrinter.java   | 28 ++++++++-----------
 src/main/java/propertyFiles/Bundle.properties |  1 +
 3 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
index bc7b34ee8b7..b9472768350 100644
--- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
@@ -17,6 +17,7 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.MailUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.json.JsonPrinter;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 import java.io.UnsupportedEncodingException;
@@ -33,6 +34,7 @@
 import javax.annotation.Resource;
 import javax.ejb.EJB;
 import javax.ejb.Stateless;
+import javax.json.JsonObject;
 import javax.mail.Address;
 import javax.mail.Message;
 import javax.mail.MessagingException;
@@ -386,11 +388,20 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                 return messageText += dataverseCreatedMessage;
             case REQUESTFILEACCESS:
                 DataFile datafile = (DataFile) targetObject;
+                
                 pattern = BundleUtil.getStringFromBundle("notification.email.requestFileAccess");
                 String requestorName = (requestor.getLastName() != null && requestor.getLastName() != null) ? requestor.getFirstName() + " " + requestor.getLastName() : BundleUtil.getStringFromBundle("notification.email.info.unavailable");
                 String requestorEmail = requestor.getEmail() != null ? requestor.getEmail() : BundleUtil.getStringFromBundle("notification.email.info.unavailable"); 
                 String[] paramArrayRequestFileAccess = {datafile.getOwner().getDisplayName(), requestorName, requestorEmail, getDatasetManageFileAccessLink(datafile)};
                 messageText += MessageFormat.format(pattern, paramArrayRequestFileAccess);
+                FileAccessRequest far = datafile.getAccessRequestForAssignee(requestor);
+                GuestbookResponse gbr = far.getGuestbookResponse();
+                if (gbr != null) {
+                    JsonObject gbJson = JsonPrinter.json(gbr);
+                    messageText += MessageFormat.format(
+                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"),
+                            JsonUtil.prettyPrint(gbJson));
+                }
                 return messageText;
             case GRANTFILEACCESS:
                 dataset = (Dataset) targetObject;
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 12d932879e9..e8e9b3c2a94 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -1,22 +1,6 @@
 package edu.harvard.iq.dataverse.util.json;
 
 import edu.harvard.iq.dataverse.*;
-import edu.harvard.iq.dataverse.AuxiliaryFile;
-import edu.harvard.iq.dataverse.ControlledVocabularyValue;
-import edu.harvard.iq.dataverse.DataFile;
-import edu.harvard.iq.dataverse.DataFileTag;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetDistributor;
-import edu.harvard.iq.dataverse.DatasetFieldType;
-import edu.harvard.iq.dataverse.DatasetField;
-import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
-import edu.harvard.iq.dataverse.DatasetFieldValue;
-import edu.harvard.iq.dataverse.DatasetLock;
-import edu.harvard.iq.dataverse.DatasetVersion;
-import edu.harvard.iq.dataverse.Dataverse;
-import edu.harvard.iq.dataverse.DataverseContact;
-import edu.harvard.iq.dataverse.DataverseFacet;
-import edu.harvard.iq.dataverse.DataverseTheme;
 import edu.harvard.iq.dataverse.api.Datasets;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.authorization.groups.impl.maildomain.MailDomainGroup;
@@ -1205,4 +1189,16 @@ private static JsonObjectBuilder jsonLicense(DatasetVersion dsv) {
         }
         return licenseJsonObjectBuilder;
     }
+
+    public static JsonObject json(GuestbookResponse gbr) {
+        JsonObjectBuilder gbrJsonObjectBuilder = jsonObjectBuilder()
+                .add("name", gbr.getName())
+                .add("email", gbr.getEmail())
+                .add("institution", gbr.getInstitution())
+                .add("position", gbr.getPosition());
+        for(CustomQuestionResponse cqr : gbr.getCustomQuestionResponses()) {
+            gbrJsonObjectBuilder.add(cqr.getCustomQuestion().getQuestionString(), cqr.getResponse());
+        }
+        return gbrJsonObjectBuilder.build();
+    }
 }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 4873ed3c52d..efb50ec28b8 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -770,6 +770,7 @@ notification.email.greeting.html=Hello, <br>
 notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance.
 notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0} . Note, the verify link will expire after {1}. Send another verification email by visiting your account page.
 notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3}.
+notification.email.requestFileAccess.guestbookResponse=Guestbook Response:\n\n {0}
 notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1} ).
 notification.email.rejectFileAccess=Your request for access was rejected for the requested files in the dataset: {0} (view at {1} ). If you have any questions about why your request was rejected, you may reach the dataset owner using the "Contact" link on the upper right corner of the dataset page.
 # Bundle file editors, please note that "notification.email.createDataverse" is used in a unit test

From e37547b519d424696a8afa6e90f0960069e4f114 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 15:56:23 -0400
Subject: [PATCH 239/922] add response date to email gbr entry

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index e8e9b3c2a94..7f8a8114078 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -1192,6 +1192,7 @@ private static JsonObjectBuilder jsonLicense(DatasetVersion dsv) {
 
     public static JsonObject json(GuestbookResponse gbr) {
         JsonObjectBuilder gbrJsonObjectBuilder = jsonObjectBuilder()
+                .add("responseDate", gbr.getResponseDate())
                 .add("name", gbr.getName())
                 .add("email", gbr.getEmail())
                 .add("institution", gbr.getInstitution())

From 548cccf41136ab17da2abd6ff24fee23050dc16e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 15:56:32 -0400
Subject: [PATCH 240/922] remove unused method

---
 .../java/edu/harvard/iq/dataverse/GuestbookResponse.java     | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 5c39d1039d9..de418f7cc07 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -235,11 +235,6 @@ public void setResponseTime(Date responseTime) {
     public String getResponseDate() {
         return new SimpleDateFormat("MMMM d, yyyy").format(responseTime);
     }
-    
-    public String getResponseDateForDisplay(){
-        return null; //    SimpleDateFormat("yyyy").format(new Timestamp(new Date().getTime()));
-    }
-    
 
     public List<CustomQuestionResponse> getCustomQuestionResponses() {
         return customQuestionResponses;

From 0f0f1905e4e515897ca50cf3cb82608a8047cc8d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 15:57:05 -0400
Subject: [PATCH 241/922] set creation date in constructors, refactor
 constructors

---
 src/main/java/edu/harvard/iq/dataverse/DataFile.java     | 6 ++----
 .../java/edu/harvard/iq/dataverse/FileAccessRequest.java | 9 ++-------
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index 7b59487d3f3..dd52555fdcf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -783,6 +783,7 @@ public void addFileAccessRequester(AuthenticatedUser authenticatedUser) {
         }
 
         Set<AuthenticatedUser> existingUsers = this.fileAccessRequests.stream()
+            .filter(far -> far.isStateCreated())
             .map(FileAccessRequest::getRequester)
             .collect(Collectors.toSet());
 
@@ -790,10 +791,7 @@ public void addFileAccessRequester(AuthenticatedUser authenticatedUser) {
             return;
         }
 
-        FileAccessRequest request = new FileAccessRequest();
-        request.setCreationTime(new Date());
-        request.setDataFile(this);
-        request.setRequester(authenticatedUser);
+        FileAccessRequest request = new FileAccessRequest(this, authenticatedUser);
 
         this.fileAccessRequests.add(request);
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
index ddde8049dff..e5d74ee09dc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
@@ -77,21 +77,16 @@ public enum RequestState {CREATED, GRANTED, REJECTED};
     @Column(name = "creation_time")
     private Date creationTime;
     
-    public FileAccessRequest(){
-        
-    }
-    
     public FileAccessRequest(DataFile df, AuthenticatedUser au){
         setDataFile(df);
         setRequester(au);
         setState(RequestState.CREATED);
+        setCreationTime(new Date());
     }
     
     public FileAccessRequest(DataFile df, AuthenticatedUser au, GuestbookResponse gbr){
-        setDataFile(df);
-        setRequester(au);
+        this(df, au);
         setGuestbookResponse(gbr);
-        setState(RequestState.CREATED);
     }
      
     public Long getId() {

From 4dff0493cf6c3b52024563bff9a5a4505da6dcdb Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 18 Jul 2023 16:36:22 -0400
Subject: [PATCH 242/922] restore no-param constructor, cleanup

---
 .../iq/dataverse/FileAccessRequest.java       |  3 ++
 .../iq/dataverse/FileDownloadHelper.java      |  9 +---
 .../command/impl/RequestAccessCommand.java    | 49 ++++++++++---------
 3 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
index e5d74ee09dc..6261ef4b46e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequest.java
@@ -77,6 +77,9 @@ public enum RequestState {CREATED, GRANTED, REJECTED};
     @Column(name = "creation_time")
     private Date creationTime;
     
+    public FileAccessRequest(){
+    }
+    
     public FileAccessRequest(DataFile df, AuthenticatedUser au){
         setDataFile(df);
         setRequester(au);
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index 7ec8272ac5b..e9deca0f0e5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -336,20 +336,13 @@ private boolean processRequestAccess(DataFile file, Boolean sendNotification) {
          if (fileDownloadService.requestAccess(file.getId())) {
              // update the local file object so that the page properly updates
              AuthenticatedUser user = (AuthenticatedUser) session.getUser();
+             //This seems to be required because we don't get the updated file object back from the command called in the fileDownloadService.requestAccess call above
              file.addFileAccessRequester(user);
 
              // create notification if necessary
              if (sendNotification) {
                  fileDownloadService.sendRequestFileAccessNotification(file.getOwner(), file.getId(), (AuthenticatedUser) session.getUser());
              }
-             //ToDO QDRADA - where to write the response?
-             /*
-             //write the guestbookResponse if there is an enabled guestbook
-             GuestbookResponse gbr = this.getGuestbookResponse(); //can we be sure this is the correct guestbookResponse?? - can it get out of sync??
-             if( gbr != null && gbr.getGuestbook().isEnabled() ){
-                 fileDownloadService.writeGuestbookResponseRecordForRequestAccess(gbr); 
-             }
-             */
              JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success"));
              return true;
          }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index a8c6150b21e..f18ba249b03 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -26,30 +26,29 @@
  */
 @RequiredPermissions({})
 public class RequestAccessCommand extends AbstractCommand<DataFile> {
-    
+
     private static final Logger logger = Logger.getLogger(RequestAccessCommand.class.getName());
-    
+
     private final DataFile file;
     private final AuthenticatedUser requester;
     private final FileAccessRequest fileAccessRequest;
     private final Boolean sendNotification;
 
-
     public RequestAccessCommand(DataverseRequest dvRequest, DataFile file) {
         // for data file check permission on owning dataset
-        super(dvRequest, file);        
-        this.file = file;        
+        super(dvRequest, file);
+        this.file = file;
         this.requester = (AuthenticatedUser) dvRequest.getUser();
-        this.fileAccessRequest = new FileAccessRequest(file,requester);
+        this.fileAccessRequest = new FileAccessRequest(file, requester);
         this.sendNotification = false;
     }
-    
-        public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean sendNotification) {
+
+    public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean sendNotification) {
         // for data file check permission on owning dataset
-        super(dvRequest, file);        
-        this.file = file;        
+        super(dvRequest, file);
+        this.file = file;
         this.requester = (AuthenticatedUser) dvRequest.getUser();
-        this.fileAccessRequest = new FileAccessRequest(file,requester);
+        this.fileAccessRequest = new FileAccessRequest(file, requester);
         this.sendNotification = sendNotification;
     }
 
@@ -57,7 +56,8 @@ public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Guestbook
         this(dvRequest, file, gbr, false);
     }
 
-    public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr, Boolean sendNotification) {
+    public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, GuestbookResponse gbr,
+            Boolean sendNotification) {
         // for data file check permission on owning dataset
         super(dvRequest, file);
         this.file = file;
@@ -72,27 +72,30 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         if (!file.getOwner().isFileAccessRequest()) {
             throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed"), this);
         }
-        
-        //if user already has permission to download file or the file is public throw command exception
-        logger.info("User: " + this.getRequest().getAuthenticatedUser().getName());
-        logger.info("File: " + file.getId() + " : restricted?: " + file.isRestricted());
-        logger.info("permission?: " + ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile));
-        if (!file.isRestricted() || ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)) {
-            throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.alreadyHasDownloadPermisssion"), this);
+
+        // if user already has permission to download file or the file is public throw
+        // command exception
+        logger.fine("User: " + this.getRequest().getAuthenticatedUser().getName());
+        logger.fine("File: " + file.getId() + " : restricted?: " + file.isRestricted());
+        logger.fine(
+                "permission?: " + ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile));
+        if (!file.isRestricted()
+                || ctxt.permissions().requestOn(this.getRequest(), file).has(Permission.DownloadFile)) {
+            throw new CommandException(
+                    BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.alreadyHasDownloadPermisssion"),
+                    this);
         }
 
-        if(FileUtil.isActivelyEmbargoed(file)) {
+        if (FileUtil.isActivelyEmbargoed(file)) {
             throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.embargoed"), this);
         }
         file.getFileAccessRequests().add(fileAccessRequest);
-        file.addFileAccessRequester(requester);
         requester.getFileAccessRequests().add(fileAccessRequest);
         if (sendNotification) {
-            logger.info("ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);"); 
+            logger.fine("ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);");
             ctxt.fileDownload().sendRequestFileAccessNotification(this.file.getOwner(), this.file.getId(), requester);
         }
         return ctxt.files().save(file);
     }
 
 }
-

From 9a45dcb826e839558b175cb3e2972720aca0f172 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 19 Jul 2023 12:37:33 -0400
Subject: [PATCH 243/922] make sure arrays exist, don't recreate request,
 refactor

---
 .../java/edu/harvard/iq/dataverse/DataFile.java | 13 +------------
 .../iq/dataverse/FileDownloadHelper.java        |  3 ++-
 .../command/impl/RequestAccessCommand.java      | 17 ++++++++++-------
 3 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index dd52555fdcf..4e2b43a0bd9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -777,22 +777,11 @@ public void setFileAccessRequesters(List<AuthenticatedUser> fileAccessRequesters
     }
 
 
-    public void addFileAccessRequester(AuthenticatedUser authenticatedUser) {
+    public void addFileAccessRequest(FileAccessRequest request) {
         if (this.fileAccessRequests == null) {
             this.fileAccessRequests = new ArrayList<>();
         }
 
-        Set<AuthenticatedUser> existingUsers = this.fileAccessRequests.stream()
-            .filter(far -> far.isStateCreated())
-            .map(FileAccessRequest::getRequester)
-            .collect(Collectors.toSet());
-
-        if (existingUsers.contains(authenticatedUser)) {
-            return;
-        }
-
-        FileAccessRequest request = new FileAccessRequest(this, authenticatedUser);
-
         this.fileAccessRequests.add(request);
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index e9deca0f0e5..6a999bc9fa9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -337,7 +337,8 @@ private boolean processRequestAccess(DataFile file, Boolean sendNotification) {
              // update the local file object so that the page properly updates
              AuthenticatedUser user = (AuthenticatedUser) session.getUser();
              //This seems to be required because we don't get the updated file object back from the command called in the fileDownloadService.requestAccess call above
-             file.addFileAccessRequester(user);
+             FileAccessRequest request = new FileAccessRequest(file, user);
+             file.addFileAccessRequest(request);
 
              // create notification if necessary
              if (sendNotification) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index f18ba249b03..27691fab0c7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -5,6 +5,8 @@
  */
 package edu.harvard.iq.dataverse.engine.command.impl;
 
+import java.util.Arrays;
+import java.util.List;
 import java.util.logging.Logger;
 
 import edu.harvard.iq.dataverse.DataFile;
@@ -36,11 +38,7 @@ public class RequestAccessCommand extends AbstractCommand<DataFile> {
 
     public RequestAccessCommand(DataverseRequest dvRequest, DataFile file) {
         // for data file check permission on owning dataset
-        super(dvRequest, file);
-        this.file = file;
-        this.requester = (AuthenticatedUser) dvRequest.getUser();
-        this.fileAccessRequest = new FileAccessRequest(file, requester);
-        this.sendNotification = false;
+        this(dvRequest, file, false);
     }
 
     public RequestAccessCommand(DataverseRequest dvRequest, DataFile file, Boolean sendNotification) {
@@ -89,8 +87,13 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         if (FileUtil.isActivelyEmbargoed(file)) {
             throw new CommandException(BundleUtil.getStringFromBundle("file.requestAccess.notAllowed.embargoed"), this);
         }
-        file.getFileAccessRequests().add(fileAccessRequest);
-        requester.getFileAccessRequests().add(fileAccessRequest);
+        file.addFileAccessRequest(fileAccessRequest);
+        List<FileAccessRequest> fars = requester.getFileAccessRequests();
+        if(fars!=null) {
+            fars.add(fileAccessRequest);
+        } else {
+            requester.setFileAccessRequests(Arrays.asList(fileAccessRequest));
+        }
         if (sendNotification) {
             logger.fine("ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);");
             ctxt.fileDownload().sendRequestFileAccessNotification(this.file.getOwner(), this.file.getId(), requester);

From 76a0dfd8b42a594010ba6ffa83e2530f5769d65b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 24 Jul 2023 12:59:00 -0400
Subject: [PATCH 244/922] blockDatasetForm doesn't exist in file page

---
 src/main/webapp/file.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index 772c4ebbc82..c66d5d8da91 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -660,7 +660,7 @@
                             </h:outputFormat>
                         </p>
                         <div class="button-block">
-                            <button class="btn btn-default" onclick="PF('accessSignUpLogIn_popup').hide();PF('blockDatasetForm').hide();" type="button">
+                            <button class="btn btn-default" onclick="PF('accessSignUpLogIn_popup').hide();" type="button">
                                 #{bundle.close}
                             </button>
                         </div>

From b7a81b1a9559a425702ca478ecbbc327532e0486 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 24 Jul 2023 16:51:38 -0400
Subject: [PATCH 245/922] fix multi-file request access

---
 .../java/edu/harvard/iq/dataverse/DatasetPage.java    | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index fe700943dd7..d47c557f381 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3247,6 +3247,8 @@ private boolean filterSelectedFiles(){
         setSelectedRestrictedFiles(new ArrayList<>());
         setSelectedUnrestrictedFiles(new ArrayList<>());
 
+        logger.info("Filter: " + this.selectedFiles.size());
+
         boolean someFiles = false;
         for (FileMetadata fmd : this.selectedFiles){
             if(this.fileDownloadHelper.canDownloadFile(fmd)){
@@ -3275,7 +3277,7 @@ public void validateFilesForRequestAccess(){
         }
 
         if(!this.selectedRestrictedFiles.isEmpty()){
-            ArrayList nonDownloadableRestrictedFiles = new ArrayList<>();
+            ArrayList<FileMetadata> nonDownloadableRestrictedFiles = new ArrayList<>();
 
             List<DataFile> userRequestedDataFiles = ((AuthenticatedUser) session.getUser()).getRequestedDataFiles();
 
@@ -3288,12 +3290,7 @@ public void validateFilesForRequestAccess(){
             if(!nonDownloadableRestrictedFiles.isEmpty()){
                 guestbookResponse.setDataFile(null);
                 guestbookResponse.setSelectedFileIds(this.getFilesIdsString(nonDownloadableRestrictedFiles));
-
-                if(this.isGuestbookAndTermsPopupRequired()){ //need to pop up the guestbook and terms dialog
-                    PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');");
-                } else {
-                    this.requestAccessMultipleFiles();
-                }
+                this.requestAccessMultipleFiles();
             } else {
                 //popup select data files
             }

From 2ffec04318861c5e12f63e100514cda1c793f41d Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Tue, 25 Jul 2023 10:54:15 -0400
Subject: [PATCH 246/922] #9724 linking a dataset requires Publish Dataset
 permission

---
 doc/sphinx-guides/source/user/dataverse-management.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst
index b5e8d8f4fc9..b039c6c65b1 100755
--- a/doc/sphinx-guides/source/user/dataverse-management.rst
+++ b/doc/sphinx-guides/source/user/dataverse-management.rst
@@ -212,7 +212,7 @@ Dataset linking allows a Dataverse collection owner to "link" their Dataverse co
 
 For example, researchers working on a collaborative study across institutions can each link their own individual institutional Dataverse collections to the one collaborative dataset, making it easier for interested parties from each institution to find the study.
 
-In order to link a dataset, you will need your account to have the "Add Dataset" permission on the Dataverse collection that is doing the linking. If you created the Dataverse collection then you should have this permission already, but if not then you will need to ask the admin of that Dataverse collection to assign that permission to your account. You do not need any special permissions on the dataset being linked.
+In order to link a dataset, you will need your account to have the "Publish Dataset" permission on the Dataverse collection that is doing the linking. If you created the Dataverse collection then you should have this permission already, but if not then you will need to ask the admin of that Dataverse collection to assign that permission to your account. You do not need any special permissions on the dataset being linked.
 
 To link a dataset to your Dataverse collection, you must navigate to that dataset and click the white "Link" button in the upper-right corner of the dataset page. This will open up a window where you can type in the name of the Dataverse collection that you would like to link the dataset to. Select your Dataverse collection and click the save button. This will establish the link, and the dataset will now appear under your Dataverse collection.
 

From 2cb77d40f4ee2a2d04f8f2578f420d2579b27564 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 25 Jul 2023 17:36:21 +0100
Subject: [PATCH 247/922] Changed: more realistic test tab file content

---
 src/test/resources/tab/test.tab | 27 +++++++++++----------------
 1 file changed, 11 insertions(+), 16 deletions(-)

diff --git a/src/test/resources/tab/test.tab b/src/test/resources/tab/test.tab
index 628a309a530..d750d42d995 100644
--- a/src/test/resources/tab/test.tab
+++ b/src/test/resources/tab/test.tab
@@ -1,16 +1,11 @@
-test1 test2 test3 test4
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
-"test1 test2 test3 test4"
+position	 name	 age
+1	"Belle"	36
+2	"Lola"	37
+3	"Jayden"	45
+4	"Margaret"	37
+5	"Russell"	40
+6	"Bertie"	60
+7	"Maud"	34
+8	"Mabel"	31
+9	"Trevor"	51
+10	"Duane"	26

From c86a1d7a32107589f7cc86fca7d0be05525edbcf Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 25 Jul 2023 17:44:36 +0100
Subject: [PATCH 248/922] Refactor: file metadatas query formatting

---
 .../edu/harvard/iq/dataverse/DatasetVersionServiceBean.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 45edfb01777..687247156b2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -50,7 +50,9 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm WHERE fm.datasetVersion.id=:datasetVersionId ORDER BY fm.label";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
+            + " WHERE fm.datasetVersion.id=:datasetVersionId"
+            + " ORDER BY fm.label";
     private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
             + " WHERE fm.datasetVersion.id = :datasetVersionId"
             + " AND fm.dataFile.id = dvo.id"

From 78060305105a9f68777f96d42181cbb469b5b9ff Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 12:45:08 -0400
Subject: [PATCH 249/922] fix dataset page download all buttons

---
 src/main/webapp/dataset.xhtml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index aa742f8e6a5..47ca447da43 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -183,6 +183,7 @@
                                                                         <h:outputFormat value="#{bundle['dataset.accessBtn.download.size']}">
                                                                             <f:param value="#{DatasetPage.sizeOfDataset}" />
                                                                         </h:outputFormat>
+                                                                        <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                                                                     </p:commandLink>
                                                                 </li>
 
@@ -197,6 +198,7 @@
                                                                             <h:outputFormat value="#{bundle['dataset.accessBtn.download.size']}">
                                                                                 <f:param value="#{DatasetPage.sizeOfDatasetOrig}" />
                                                                             </h:outputFormat>
+                                                                            <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                                                                         </p:commandLink>
                                                                     </li>
                                                                     <!-- Original Format too big -->

From 991641b2f9355196dbab361752577c84eb5a8c39 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 25 Jul 2023 18:09:33 +0100
Subject: [PATCH 250/922] Refactor: using TypedQuery instead of Query for
 getFileMetadatas

---
 .../edu/harvard/iq/dataverse/DatasetVersionServiceBean.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 687247156b2..807ba94a0a6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -1249,7 +1249,7 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
      * @return a FileMetadata list of the specified DatasetVersion
      */
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
-        Query query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria))
+        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
                 .setParameter("datasetVersionId", datasetVersion.getId());
         if (limit != null) {
             query.setMaxResults(limit);

From 14c41ce942fcf41e0f1191110e86836ad8b6a934 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 16:06:58 -0400
Subject: [PATCH 251/922] fix updating messages

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 55e11c9c386..8b89bb6107d 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -304,14 +304,14 @@
         <!--TODO - add "Accept and" if ToA exist?-->
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{!guestbookPopupRequiredAtDownload}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"
-                    update="@([id$=Messages])"
+                    update="@([id$=messagePanel])"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
                     <f:param name="DO_GB_VALIDATION" value="true"/>
             </p:commandLink>
 
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{guestbookPopupRequiredAtDownload}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.requestAccessIndirect()}"
-                    update="@([id$=Messages])"
+                    update="@([id$=messagePanel])"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
                     <f:param name="DO_GB_VALIDATION" value="true"/>
             </p:commandLink>

From 90fba4dab21a80784e460d3011ce4843c0b94123 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 16:07:26 -0400
Subject: [PATCH 252/922] switch to formatted text

---
 .../iq/dataverse/GuestbookResponse.java       | 27 +++++++++++++++++++
 .../iq/dataverse/util/json/JsonPrinter.java   | 12 ---------
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index de418f7cc07..3c70b1103c7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -310,6 +310,33 @@ public void setSessionId(String sessionId) {
         this.fileDownload.setSessionId(sessionId);
     }
     
+    public String toFormattedResponse() {
+
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("\nGuestbook Response ID: " + getId() + "\n");
+        sb.append("Response Date: " + getResponseDate() + "\n");
+        sb.append("Requestor\n  Name: " + getName() + "\n");
+        sb.append("  Email: " + getEmail() + "\n");
+        sb.append("  Institution: " + wrapNullAnswer(getInstitution()) + "\n");
+        sb.append("  Position: " + wrapNullAnswer(getPosition()) + "\n");
+        sb.append("  Custom Question Responses:\n");
+
+        for (CustomQuestionResponse cqr : getCustomQuestionResponses()) {
+            sb.append("    Q: " + cqr.getCustomQuestion().getQuestionString() + "\n    A: " + wrapNullAnswer(cqr.getResponse())
+                    + "\n\n");
+        }
+        return sb.toString();
+    }
+    
+    private String wrapNullAnswer(String answer) {
+        //This assumes we don't have to distinguish null from when the user actually answers "(No Reponse)". The db still has the real value
+        if (answer == null) {
+            return "(No Reponse)";
+        }
+        return answer;
+    }
+    
     @Override
     public int hashCode() {
         int hash = 0;
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index aa641380936..3bfb5d816de 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -1195,16 +1195,4 @@ private static JsonObjectBuilder jsonLicense(DatasetVersion dsv) {
         return licenseJsonObjectBuilder;
     }
 
-    public static JsonObject json(GuestbookResponse gbr) {
-        JsonObjectBuilder gbrJsonObjectBuilder = jsonObjectBuilder()
-                .add("responseDate", gbr.getResponseDate())
-                .add("name", gbr.getName())
-                .add("email", gbr.getEmail())
-                .add("institution", gbr.getInstitution())
-                .add("position", gbr.getPosition());
-        for(CustomQuestionResponse cqr : gbr.getCustomQuestionResponses()) {
-            gbrJsonObjectBuilder.add(cqr.getCustomQuestion().getQuestionString(), cqr.getResponse());
-        }
-        return gbrJsonObjectBuilder.build();
-    }
 }

From adc106ba3195dc55f2a1ddd9be5d0763643a1e38 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 16:08:28 -0400
Subject: [PATCH 253/922] Add email/notification to requestor, add page
 messages for other routes

---
 .../iq/dataverse/FileDownloadHelper.java      |  6 ++-
 .../iq/dataverse/FileDownloadServiceBean.java | 38 ++++---------------
 .../harvard/iq/dataverse/MailServiceBean.java | 19 ++++++++--
 .../UserNotificationServiceBean.java          | 21 ----------
 .../providers/builtin/DataverseUserPage.java  |  1 +
 src/main/java/propertyFiles/Bundle.properties |  3 ++
 src/main/webapp/dataverseuser.xhtml           |  8 ++++
 7 files changed, 40 insertions(+), 56 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index 6a999bc9fa9..dc16da87258 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -318,7 +318,9 @@ public void requestAccessMultiple(List<DataFile> files) {
              }
          }
          if (notificationFile != null && succeeded) {
-             fileDownloadService.sendRequestFileAccessNotification(notificationFile.getOwner(), notificationFile.getId(), (AuthenticatedUser) session.getUser());
+             fileDownloadService.sendRequestFileAccessNotification(notificationFile.getOwner(),
+                     notificationFile.getId(), (AuthenticatedUser) session.getUser());
+             JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success"));
          }
      }
     
@@ -343,8 +345,8 @@ private boolean processRequestAccess(DataFile file, Boolean sendNotification) {
              // create notification if necessary
              if (sendNotification) {
                  fileDownloadService.sendRequestFileAccessNotification(file.getOwner(), file.getId(), (AuthenticatedUser) session.getUser());
+                 JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success"));
              }
-             JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success"));
              return true;
          }
          JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("file.accessRequested.alreadyRequested", Arrays.asList(file.getDisplayName())));
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 6341f00d00c..b7200bcdea3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -20,6 +20,7 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
+import edu.harvard.iq.dataverse.util.JsfHelper;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import java.io.IOException;
 import java.sql.Timestamp;
@@ -217,14 +218,16 @@ public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookRe
             writeGuestbookResponseRecordForRequestAccess(guestbookResponse);
             if(requestAccess(dataFile,guestbookResponse)){
                 countRequestAccessSuccess++;
+            } else {
+                JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("file.accessRequested.alreadyRequested", Arrays.asList(dataFile.getDisplayName())));
             }
         }
 
         if(countRequestAccessSuccess > 0){
             DataFile firstDataFile = selectedDataFiles.get(0);
             sendRequestFileAccessNotification(firstDataFile.getOwner(), firstDataFile.getId(), (AuthenticatedUser) session.getUser());
+            JsfHelper.addSuccessMessage(BundleUtil.getStringFromBundle("file.accessRequested.success"));
         }
-
     }
 
     public void writeGuestbookResponseRecord(GuestbookResponse guestbookResponse, FileMetadata fileMetadata, String format) {
@@ -566,40 +569,15 @@ public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){
     }
     
     public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) {
+        Timestamp ts = new Timestamp(new Date().getTime());
         permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> {
-            userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, false);
+            userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, false);
         });
+        //send the user that requested access a notification that they requested the access
+        userNotificationService.sendNotification((AuthenticatedUser) session.getUser(), ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId, null, requestor, false);
 
     } 
     
-    public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, GuestbookResponse gb){
-        Timestamp ts = new Timestamp(new Date().getTime()); 
-        UserNotification un = null;
-
-        //String appendMsgText = (gb == null)?("") : this.getGuestbookAppendEmailDetails(gb);
-        String appendMsgText = "";
-
-        //first send a notification for all the Users that have ManageDatasetPermissions a notification that a user has requested accedd    
-        List<AuthenticatedUser> mngDsPermUsers = permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset);
-
-        for (AuthenticatedUser au : mngDsPermUsers){
-            un = userNotificationService.sendUserNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId);
-
-            if(un != null){
-
-               boolean mailed = mailService.sendNotificationEmail(un, appendMsgText, (AuthenticatedUser)session.getUser(),false);
-               if(mailed){
-                   un.setEmailed(true);
-                   userNotificationService.save(un);
-               }    
-            }
-        }
-
-        //send the user that requested access a notification that they requested the access
-        userNotificationService.sendNotification((AuthenticatedUser) session.getUser(), ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId);
-    }
- 
-    
     public String generateServiceKey() {
         UUID uid = UUID.randomUUID();
         // last 8 bytes, of the random UUID, 16 hex digits: 
diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
index b9472768350..5b69641f029 100644
--- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
@@ -387,6 +387,7 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                 logger.fine(dataverseCreatedMessage);
                 return messageText += dataverseCreatedMessage;
             case REQUESTFILEACCESS:
+                //Notification to those who can grant file access requests on a dataset when a user makes a request
                 DataFile datafile = (DataFile) targetObject;
                 
                 pattern = BundleUtil.getStringFromBundle("notification.email.requestFileAccess");
@@ -397,10 +398,8 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                 FileAccessRequest far = datafile.getAccessRequestForAssignee(requestor);
                 GuestbookResponse gbr = far.getGuestbookResponse();
                 if (gbr != null) {
-                    JsonObject gbJson = JsonPrinter.json(gbr);
                     messageText += MessageFormat.format(
-                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"),
-                            JsonUtil.prettyPrint(gbJson));
+                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toFormattedResponse());
                 }
                 return messageText;
             case GRANTFILEACCESS:
@@ -644,6 +643,19 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                         dataset.getDisplayName()};
                 messageText = MessageFormat.format(pattern, paramArrayDatasetMentioned);
                 return messageText;
+            case REQUESTEDFILEACCESS:
+                //Notification to requestor when they make a request
+                datafile = (DataFile) targetObject;
+                
+                pattern = BundleUtil.getStringFromBundle("notification.email.requestedFileAccess");
+                messageText += MessageFormat.format(pattern, getDvObjectLink(datafile), datafile.getOwner().getDisplayName());
+                far = datafile.getAccessRequestForAssignee(requestor);
+                gbr = far.getGuestbookResponse();
+                if (gbr != null) {
+                    messageText += MessageFormat.format(
+                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toFormattedResponse());
+                }
+                return messageText;
         }
 
         return "";
@@ -664,6 +676,7 @@ public Object getObjectOfNotification (UserNotification userNotification){
             case CREATEDV:
                 return dataverseService.find(userNotification.getObjectId());
             case REQUESTFILEACCESS:
+            case REQUESTEDFILEACCESS:
                 return dataFileService.find(userNotification.getObjectId());
             case GRANTFILEACCESS:
             case REJECTFILEACCESS:
diff --git a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java
index 972f26f6830..4ca3b19b708 100644
--- a/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/UserNotificationServiceBean.java
@@ -132,27 +132,6 @@ public void sendNotification(AuthenticatedUser dataverseUser, Timestamp sendDate
         }
     }
     
-    /**
-     * Returns a UserNotification that was sent to a dataverseUser.
-     * Sends ONLY the UserNotification (no email is sent via this method).
-     * All parameters are assumed to be valid, non-null objects.
-     * 
-     * @param dataverseUser - the AuthenticatedUser to whom the notification is to be sent
-     * @param sendDate - the time and date the notification was sent.
-     * @param type - the type of notification to be sent (see UserNotification for the different types)
-     * @param objectId -  the ID of the Dataverse object (Dataverse, Dataset, etc.) that the notification pertains to 
-     * @return The UserNotification that was sent to the dataverseUser
-     */
-
-    public UserNotification sendUserNotification(AuthenticatedUser dataverseUser, Timestamp sendDate, Type type, Long objectId) {
-        UserNotification userNotification = new UserNotification();
-        userNotification.setUser(dataverseUser);
-        userNotification.setSendDate(sendDate);
-        userNotification.setType(type);
-        userNotification.setObjectId(objectId);
-        this.save(userNotification);
-        return userNotification;
-    }
 
     public boolean isEmailMuted(UserNotification userNotification) {
         final Type type = userNotification.getType();
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java
index 5c0f3a49f76..a27713d09e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/builtin/DataverseUserPage.java
@@ -488,6 +488,7 @@ public void displayNotification() {
                     break;
 
                 case REQUESTFILEACCESS:
+                case REQUESTEDFILEACCESS:
                     DataFile file = fileService.find(userNotification.getObjectId());
                     if (file != null) {
                         userNotification.setTheObject(file.getOwner());
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index efb50ec28b8..1116ec3a006 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -206,6 +206,7 @@ notification.welcome=Welcome to {0}! Get started by adding or finding data. Have
 notification.welcomeConfirmEmail=Also, check for your welcome email to verify your address.
 notification.demoSite=Demo Site
 notification.requestFileAccess=File access requested for dataset: {0} was made by {1} ({2}).
+notification.requestedFileAccess=You have requested access to files in dataset: {0}.
 notification.grantFileAccess=Access granted for files in dataset: {0}.
 notification.rejectFileAccess=Access rejected for requested files in dataset: {0}.
 notification.createDataverse={0} was created in {1} . To learn more about what you can do with your dataverse, check out the {2}. 
@@ -745,6 +746,7 @@ notification.email.create.dataverse.subject={0}: Your dataverse has been created
 notification.email.create.dataset.subject={0}: Dataset "{1}" has been created
 notification.email.dataset.created.subject={0}: Dataset "{1}" has been created
 notification.email.request.file.access.subject={0}: Access has been requested for a restricted file
+notification.email.requested.file.access.subject={0}: You have requested access to a restricted file
 notification.email.grant.file.access.subject={0}: You have been granted access to a restricted file
 notification.email.rejected.file.access.subject={0}: Your request for access to a restricted file has been rejected
 notification.email.submit.dataset.subject={0}: Dataset "{1}" has been submitted for review
@@ -789,6 +791,7 @@ notification.email.changeEmail=Hello, {0}.{1}\n\nPlease contact us if you did no
 notification.email.passwordReset=Hi {0},\n\nSomeone, hopefully you, requested a password reset for {1}.\n\nPlease click the link below to reset your Dataverse account password:\n\n {2} \n\n The link above will only work for the next {3} minutes.\n\n Please contact us if you did not request this password reset or need further help.
 notification.email.passwordReset.subject=Dataverse Password Reset Requested
 notification.email.datasetWasCreated=Dataset "<a href = "{0}">{1}</a>" was just created by {2} in the {3} collection.
+notification.email.requestedFileAccess=You have requested access to a file(s) in dataset "<a href = "{0}">{1}</a>". Your request has been sent to the managers of this dataset who will grant or reject your request. If you have any questions, you may reach the dataset managers using the "Contact" link on the upper right corner of the dataset page.
 hours=hours
 hour=hour
 minutes=minutes
diff --git a/src/main/webapp/dataverseuser.xhtml b/src/main/webapp/dataverseuser.xhtml
index 51f5bfa9f8a..2426cf980d3 100644
--- a/src/main/webapp/dataverseuser.xhtml
+++ b/src/main/webapp/dataverseuser.xhtml
@@ -236,6 +236,14 @@
                                                         </o:param>
                                                     </h:outputFormat>
                                                 </ui:fragment>
+                                                <ui:fragment rendered="#{item.type == 'REQUESTEDFILEACCESS'}">
+                                                    <span class="icon-dataset text-icon-inline text-muted"></span>
+                                                    <h:outputFormat value="#{bundle['notification.requestedFileAccess']}" escape="false">
+                                                        <o:param>
+                                                            <a href="/dataset.xhtml?persistentId=#{item.theObject.getGlobalId()}" title="#{item.theObject.displayName}">#{item.theObject.displayName}</a>
+                                                        </o:param>
+                                                    </h:outputFormat>
+                                                </ui:fragment>
                                                 <ui:fragment rendered="#{item.type == 'GRANTFILEACCESS'}">
                                                     <span class="icon-dataset text-icon-inline text-muted"></span>
                                                     <h:outputFormat value="#{bundle['notification.grantFileAccess']}" escape="false">

From e36ac4a8e2043782d71e665fe40080d264313592 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 16:30:02 -0400
Subject: [PATCH 254/922] i18n gb text

---
 .../iq/dataverse/GuestbookResponse.java       | 25 ++++++++++++-------
 src/main/java/propertyFiles/Bundle.properties | 11 +++++++-
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 3c70b1103c7..52b0f10c891 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -8,6 +8,8 @@
 
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.externaltools.ExternalTool;
+import edu.harvard.iq.dataverse.util.BundleUtil;
+
 import java.io.Serializable;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -314,17 +316,22 @@ public String toFormattedResponse() {
 
         StringBuilder sb = new StringBuilder();
 
-        sb.append("\nGuestbook Response ID: " + getId() + "\n");
-        sb.append("Response Date: " + getResponseDate() + "\n");
-        sb.append("Requestor\n  Name: " + getName() + "\n");
-        sb.append("  Email: " + getEmail() + "\n");
-        sb.append("  Institution: " + wrapNullAnswer(getInstitution()) + "\n");
-        sb.append("  Position: " + wrapNullAnswer(getPosition()) + "\n");
-        sb.append("  Custom Question Responses:\n");
+        sb.append("\n" + BundleUtil.getStringFromBundle("dataset.guestbookResponse.id") + ": " + getId() + "\n");
+        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.date") + ": " + getResponseDate() + "\n");
+        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.respondent") + "\n  "
+                + BundleUtil.getStringFromBundle("name") + ": " + getName() + "\n");
+        sb.append("  " + BundleUtil.getStringFromBundle("email") + ": " + getEmail() + "\n");
+        sb.append(
+                "  " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "\n");
+        sb.append("  " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "\n");
+        sb.append("  " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.guestbook.additionalQuestions")
+                + ":\n");
 
         for (CustomQuestionResponse cqr : getCustomQuestionResponses()) {
-            sb.append("    Q: " + cqr.getCustomQuestion().getQuestionString() + "\n    A: " + wrapNullAnswer(cqr.getResponse())
-                    + "\n\n");
+            sb.append("    " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": "
+                    + cqr.getCustomQuestion().getQuestionString() + "\n    "
+                    + BundleUtil.getStringFromBundle("dataset.guestbookResponse.answer") + ": "
+                    + wrapNullAnswer(cqr.getResponse()) + "\n\n");
         }
         return sb.toString();
     }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 1116ec3a006..69fe6e2e87b 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -772,7 +772,7 @@ notification.email.greeting.html=Hello, <br>
 notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance.
 notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0} . Note, the verify link will expire after {1}. Send another verification email by visiting your account page.
 notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3}.
-notification.email.requestFileAccess.guestbookResponse=Guestbook Response:\n\n {0}
+notification.email.requestFileAccess.guestbookResponse=\n\nGuestbook Response:\n\n {0}
 notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1} ).
 notification.email.rejectFileAccess=Your request for access was rejected for the requested files in the dataset: {0} (view at {1} ). If you have any questions about why your request was rejected, you may reach the dataset owner using the "Contact" link on the upper right corner of the dataset page.
 # Bundle file editors, please note that "notification.email.createDataverse" is used in a unit test
@@ -1367,6 +1367,15 @@ dataset.guestbookResponse.guestbook.additionalQuestions=Additional Questions
 dataset.guestbookResponse.showPreview.errorMessage=Can't show preview.
 dataset.guestbookResponse.showPreview.errorDetail=Couldn't write guestbook response.
 
+#GuestbookResponse
+dataset.guestbookResponse=Guestbook Response
+dataset.guestbookResponse.id=Guestbook Response ID
+dataset.guestbookResponse.date=Response Date
+dataset.guestbookResponse.respondent=Respondent
+dataset.guestbookResponse.question=Q
+dataset.guestbookResponse.answer=A
+
+
 # dataset.xhtml
 dataset.configureBtn=Configure
 dataset.pageTitle=Add New Dataset

From 606b33b707a2f491371223a6447e8083cf45eeea Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 16:36:43 -0400
Subject: [PATCH 255/922] i18n noResponse, change indenting

---
 .../java/edu/harvard/iq/dataverse/GuestbookResponse.java  | 8 ++++----
 src/main/java/propertyFiles/Bundle.properties             | 1 +
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 52b0f10c891..60205eeabcf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -324,12 +324,12 @@ public String toFormattedResponse() {
         sb.append(
                 "  " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "\n");
         sb.append("  " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "\n");
-        sb.append("  " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.guestbook.additionalQuestions")
+        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.guestbook.additionalQuestions")
                 + ":\n");
 
         for (CustomQuestionResponse cqr : getCustomQuestionResponses()) {
-            sb.append("    " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": "
-                    + cqr.getCustomQuestion().getQuestionString() + "\n    "
+            sb.append("  " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": "
+                    + cqr.getCustomQuestion().getQuestionString() + "\n  "
                     + BundleUtil.getStringFromBundle("dataset.guestbookResponse.answer") + ": "
                     + wrapNullAnswer(cqr.getResponse()) + "\n\n");
         }
@@ -339,7 +339,7 @@ public String toFormattedResponse() {
     private String wrapNullAnswer(String answer) {
         //This assumes we don't have to distinguish null from when the user actually answers "(No Reponse)". The db still has the real value
         if (answer == null) {
-            return "(No Reponse)";
+            return BundleUtil.getStringFromBundle("dataset.guestbookResponse.noResponse");
         }
         return answer;
     }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 69fe6e2e87b..6aed60088b2 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1374,6 +1374,7 @@ dataset.guestbookResponse.date=Response Date
 dataset.guestbookResponse.respondent=Respondent
 dataset.guestbookResponse.question=Q
 dataset.guestbookResponse.answer=A
+dataset.guestbookResponse.noResponse=(No Response)
 
 
 # dataset.xhtml

From 953fb5b5c82d80927228103ad8a7b2460bd8f682 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 16:42:36 -0400
Subject: [PATCH 256/922] don't show gb at download for
 isFileAccessRequest=false case and gbar

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 6 ++----
 src/main/java/edu/harvard/iq/dataverse/FilePage.java    | 6 ++----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index d47c557f381..8dda1448e1d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5263,10 +5263,8 @@ public boolean isTermsPopupRequired(){
     }
     
     public boolean isGuestbookPopupRequiredAtDownload(){
-        // Only show guestbookAtDownload if fileAccessRequest is turned off for the
-        // dataset or guestbook at request is disabled (legacy behavior)
-        return isGuestbookPopupRequired() && (!workingVersion.getDataset().isFileAccessRequest()
-                || !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest());
+        // Only show guestbookAtDownload if guestbook at request is disabled (legacy behavior)
+        return isGuestbookPopupRequired() && !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest();
     }
     
     public String requestAccessMultipleFiles() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
index fa8b494bce2..2db5abc0049 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
@@ -349,11 +349,9 @@ public boolean isGuestbookAndTermsPopupRequired() {
     }
     
     public boolean isGuestbookPopupRequiredAtDownload(){
-        // Only show guestbookAtDownload if fileAccessRequest is turned off for the
-        // dataset or guestbook at request is disabled (legacy behavior)
+        // Only show guestbookAtDownload if guestbook at request is disabled (legacy behavior)
         DatasetVersion workingVersion = fileMetadata.getDatasetVersion();
-        return FileUtil.isGuestbookPopupRequired(workingVersion) && (!workingVersion.getDataset().isFileAccessRequest()
-                || !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest());
+        return FileUtil.isGuestbookPopupRequired(workingVersion) && !workingVersion.getDataset().getEffectiveGuestbookEntryAtRequest();
     }
 
     public void setFileMetadata(FileMetadata fileMetadata) {

From 2c5298ea29156a73177a706e271701522361c434 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 25 Jul 2023 17:00:31 -0400
Subject: [PATCH 257/922] cleanup

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 8dda1448e1d..3aad3255e6d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -375,7 +375,6 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
 
     public void setTermsGuestbookPopupAction(String popupAction){
         if(popupAction != null && popupAction.length() > 0){
-            logger.info("TGPA set to " + popupAction);
             this.termsGuestbookPopupAction = popupAction;
         }
 
@@ -3247,8 +3246,6 @@ private boolean filterSelectedFiles(){
         setSelectedRestrictedFiles(new ArrayList<>());
         setSelectedUnrestrictedFiles(new ArrayList<>());
 
-        logger.info("Filter: " + this.selectedFiles.size());
-
         boolean someFiles = false;
         for (FileMetadata fmd : this.selectedFiles){
             if(this.fileDownloadHelper.canDownloadFile(fmd)){

From 1ff9d905d69ba7d9c3b3858f2b6df95adcdddac9 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 27 Jul 2023 17:42:06 +0100
Subject: [PATCH 258/922] Refactor: getVersionFiles endpoint invalid
 orderCriteria error handling

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java   | 4 +++-
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 7 +++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a23cecd7180..623f4798a5b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -496,11 +496,13 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
     public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         return response( req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
             try {
-                return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ)));
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
+            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 227d31e6733..282017462e4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3386,5 +3386,12 @@ public void getVersionFiles() throws IOException {
                 .body("data[2].label", equalTo(testFileName2))
                 .body("data[3].label", equalTo(testFileName3))
                 .body("data[4].label", equalTo(testFileName4));
+
+        // Test invalid order criteria
+        String invalidOrderCriteria = "invalidOrderCriteria";
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, invalidOrderCriteria, apiToken);
+        getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
+                .statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
     }
 }

From ece78ca96a8e3de4d77fe1646e95c24a607ec41c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 28 Jul 2023 10:26:13 -0400
Subject: [PATCH 259/922] add dataset name to title per request

---
 src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java | 7 +++++--
 src/main/java/propertyFiles/Bundle.properties             | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java
index d166cc753cc..88db53f48e2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/MailUtil.java
@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.util;
 
+import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.UserNotification;
@@ -39,6 +40,8 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti
                 datasetDisplayName = ((Dataset) objectOfNotification).getDisplayName();
             } else if (objectOfNotification instanceof DatasetVersion) {
                 datasetDisplayName = ((DatasetVersion) objectOfNotification).getDataset().getDisplayName();
+            } else if (objectOfNotification instanceof DataFile) {
+                datasetDisplayName = ((DataFile) objectOfNotification).getOwner().getDisplayName();
             }
         }
 
@@ -50,9 +53,9 @@ public static String getSubjectTextBasedOnNotification(UserNotification userNoti
             case CREATEDV:
                 return BundleUtil.getStringFromBundle("notification.email.create.dataverse.subject", rootDvNameAsList);
             case REQUESTFILEACCESS:
-                return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", rootDvNameAsList);
+                return BundleUtil.getStringFromBundle("notification.email.request.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName));
             case REQUESTEDFILEACCESS:
-                return BundleUtil.getStringFromBundle("notification.email.requested.file.access.subject", rootDvNameAsList);
+                return BundleUtil.getStringFromBundle("notification.email.requested.file.access.subject", Arrays.asList(rootDvNameAsList.get(0), datasetDisplayName));
             case GRANTFILEACCESS:
                 return BundleUtil.getStringFromBundle("notification.email.grant.file.access.subject", rootDvNameAsList);
             case REJECTFILEACCESS:
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 6aed60088b2..edc3c387af4 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -745,8 +745,8 @@ dashboard.card.datamove.dataset.command.error.indexingProblem=Dataset could not
 notification.email.create.dataverse.subject={0}: Your dataverse has been created
 notification.email.create.dataset.subject={0}: Dataset "{1}" has been created
 notification.email.dataset.created.subject={0}: Dataset "{1}" has been created
-notification.email.request.file.access.subject={0}: Access has been requested for a restricted file
-notification.email.requested.file.access.subject={0}: You have requested access to a restricted file
+notification.email.request.file.access.subject={0}: Access has been requested for a restricted file in dataset "{1}"
+notification.email.requested.file.access.subject={0}: You have requested access to a restricted file  in dataset "{1}"
 notification.email.grant.file.access.subject={0}: You have been granted access to a restricted file
 notification.email.rejected.file.access.subject={0}: Your request for access to a restricted file has been rejected
 notification.email.submit.dataset.subject={0}: Dataset "{1}" has been submitted for review

From d4b28c1bfe7cc74793a4fbe06ac8c49713aa13e8 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 28 Jul 2023 12:36:08 -0400
Subject: [PATCH 260/922] session.getUser is guest when using requestAccess API

---
 .../java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index b7200bcdea3..dcfe58c5830 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -574,7 +574,7 @@ public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, Auth
             userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, false);
         });
         //send the user that requested access a notification that they requested the access
-        userNotificationService.sendNotification((AuthenticatedUser) session.getUser(), ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId, null, requestor, false);
+        userNotificationService.sendNotification(requestor, ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId, null, requestor, false);
 
     } 
     

From 1775be36770c5f8a5ebe1ccbf162c29535a92e7d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 28 Jul 2023 12:36:54 -0400
Subject: [PATCH 261/922] need to have request saved before sending notice

---
 .../engine/command/impl/RequestAccessCommand.java          | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
index 27691fab0c7..bf291427341 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/RequestAccessCommand.java
@@ -94,11 +94,12 @@ public DataFile execute(CommandContext ctxt) throws CommandException {
         } else {
             requester.setFileAccessRequests(Arrays.asList(fileAccessRequest));
         }
+        DataFile savedFile = ctxt.files().save(file);
         if (sendNotification) {
-            logger.fine("ctxt.fileDownload().sendRequestFileAccessNotification(this.file, requester);");
-            ctxt.fileDownload().sendRequestFileAccessNotification(this.file.getOwner(), this.file.getId(), requester);
+            logger.fine("ctxt.fileDownload().sendRequestFileAccessNotification(savedFile, requester);");
+            ctxt.fileDownload().sendRequestFileAccessNotification(savedFile.getOwner(), savedFile.getId(), requester);
         }
-        return ctxt.files().save(file);
+        return savedFile;
     }
 
 }

From 3b2bde71b010729fd6c8bc31c6e798a3adb459bf Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 28 Jul 2023 12:46:28 -0400
Subject: [PATCH 262/922] update test

---
 src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java
index 76e447b3faa..cb22faf0316 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/MailUtilTest.java
@@ -80,7 +80,7 @@ public void testSubjectRevokeRole() {
     @Test
     public void testSubjectRequestFileAccess() {
         userNotification.setType(UserNotification.Type.REQUESTFILEACCESS);
-        assertEquals("LibraScholar: Access has been requested for a restricted file", MailUtil.getSubjectTextBasedOnNotification(userNotification, null));
+        assertEquals("LibraScholar: Access has been requested for a restricted file in dataset \"\"", MailUtil.getSubjectTextBasedOnNotification(userNotification, null));
     }
     
     @Test

From 7a9c2b383959b9608db429b7924eba387ac49a04 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 1 Aug 2023 16:28:06 +0100
Subject: [PATCH 263/922] Removed: canFileBeDownloaded endpoint and related
 logic

---
 .../iq/dataverse/FileDownloadServiceBean.java |  44 ----
 .../edu/harvard/iq/dataverse/api/Files.java   |  15 --
 .../FileDownloadServiceBeanTest.java          | 246 ------------------
 .../edu/harvard/iq/dataverse/api/FilesIT.java |  32 ---
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |   6 -
 5 files changed, 343 deletions(-)
 delete mode 100644 src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index f5bb60510cc..18980ff9eff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -572,48 +572,4 @@ public String getDirectStorageLocatrion(String storageLocation) {
             
         return null; 
     }
-
-    /**
-     * Checks if a user can download a file based on the file metadata and the permissions of the user
-     *
-     * This method is based on {@link edu.harvard.iq.dataverse.FileDownloadHelper#canDownloadFile(FileMetadata),
-     * and has been adapted to make it callable from the API instead of a view
-     *
-     * @param user requesting the download
-     * @param fileMetadata of the particular file to download
-     * @return boolean
-     */
-    public boolean canDownloadFile(User user, FileMetadata fileMetadata){
-        if (fileMetadata == null){
-            return false;
-        }
-        if ((fileMetadata.getId() == null) || (fileMetadata.getDataFile().getId() == null)){
-            return false;
-        }
-        if (user == null) {
-            return false;
-        }
-        if (user instanceof PrivateUrlUser) {
-            // Always allow download for PrivateUrlUser
-            return true;
-        }
-
-        if (fileMetadata.getDatasetVersion().isDeaccessioned()) {
-            return this.permissionService.userOn(user, fileMetadata.getDatasetVersion().getDataset()).has(Permission.EditDataset);
-        }
-
-        // Note that `isRestricted` at the FileMetadata level is for expressing intent by version. Enforcement is done with `isRestricted` at the DataFile level.
-        boolean isRestrictedFile = fileMetadata.isRestricted() || fileMetadata.getDataFile().isRestricted();
-        if (!isRestrictedFile && !FileUtil.isActivelyEmbargoed(fileMetadata)){
-            return true;
-        }
-
-        // See if the DataverseRequest, which contains IP Groups, has permission to download the file.
-        if (permissionService.requestOn(dvRequestService.getDataverseRequest(), fileMetadata.getDataFile()).has(Permission.DownloadFile)) {
-            logger.fine("The DataverseRequest (User plus IP address) has access to download the file.");
-            return true;
-        }
-
-        return false;
-    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index f42f6f5abc7..83c464f2a27 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -110,8 +110,6 @@ public class Files extends AbstractApiBean {
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
     @Inject
-    FileDownloadServiceBean fileDownloadServiceBean;
-    @Inject
     DataFileServiceBean dataFileServiceBean;
 
     private static final Logger logger = Logger.getLogger(Files.class.getName());
@@ -840,19 +838,6 @@ public Response getCountGuestbookResponses(@Context ContainerRequestContext crc,
         }, getRequestUser(crc));
     }
 
-    @GET
-    @AuthRequired
-    @Path("{id}/canBeDownloaded")
-    public Response canFileBeDownloaded(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        return ok(fileDownloadServiceBean.canDownloadFile(getRequestUser(crc), dataFile.getFileMetadata()));
-    }
-
     @GET
     @AuthRequired
     @Path("{id}/thumbnailClass")
diff --git a/src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java
deleted file mode 100644
index d754dd357a9..00000000000
--- a/src/test/java/edu/harvard/iq/dataverse/FileDownloadServiceBeanTest.java
+++ /dev/null
@@ -1,246 +0,0 @@
-package edu.harvard.iq.dataverse;
-
-import edu.harvard.iq.dataverse.authorization.Permission;
-import edu.harvard.iq.dataverse.authorization.users.User;
-import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.mockito.ArgumentMatchers;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-
-import java.time.LocalDate;
-
-import static edu.harvard.iq.dataverse.mocks.MocksFactory.makeAuthenticatedUser;
-import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-
-@ExtendWith(MockitoExtension.class)
-public class FileDownloadServiceBeanTest {
-
-    @Mock
-    private PermissionServiceBean permissionServiceBeanStub;
-    @Mock
-    private DataverseRequestServiceBean dataverseRequestServiceBeanMock;
-    @Mock
-    private MakeDataCountLoggingServiceBean makeDataCountLoggingServiceBeanMock;
-
-    private FileDownloadServiceBean sut;
-
-    private User testUser;
-
-    @BeforeEach
-    public void setUp() {
-        sut = new FileDownloadServiceBean();
-        sut.permissionService = permissionServiceBeanStub;
-        sut.dvRequestService = dataverseRequestServiceBeanMock;
-        sut.mdcLogService = makeDataCountLoggingServiceBeanMock;
-        testUser = makeAuthenticatedUser("Test", "Test");
-    }
-
-    @Test
-    public void testCanDownloadFile_withoutUser() {
-        assertFalse(sut.canDownloadFile(null, new FileMetadata()));
-    }
-
-    @Test
-    public void testCanDownloadFile_withoutFileMetadata() {
-        assertFalse(sut.canDownloadFile(testUser, null));
-    }
-
-    @Test
-    void testCanDownloadFile_withNullMetadataId() {
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(null);
-
-        assertFalse(sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @Test
-    void testCanDownloadFile_withNullDataFileId() {
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(null);
-        testFileMetadata.setDataFile(testDataFile);
-
-        assertFalse(sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @ParameterizedTest
-    @CsvSource({"false", "true"})
-    void testCanDownloadFile_forDeaccessionedFile(boolean hasPermission) {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setDataset(new Dataset());
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.DEACCESSIONED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-
-        mockPermissionResponseUserOn(hasPermission);
-
-        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @Test
-    void testCanDownloadFile_forUnrestrictedReleasedFile() {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setRestricted(false);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-
-        assertTrue(sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @Test
-    void testCanDownloadFile_forUnrestrictedReleasedActiveEmbargoFile() {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        // With an embargo, an unrestricted file should only be accessible if the embargo has ended
-
-        Embargo testEmbargo = new Embargo(LocalDate.now().plusDays(3), "Still embargoed");
-        testDataFile.setEmbargo(testEmbargo);
-
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setRestricted(false);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-        mockPermissionResponseRequestOn(false);
-
-        assertFalse(sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @Test
-    void testCanDownloadFile_forUnrestrictedReleasedExpiredEmbargoFile() {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        // With an embargo, an unrestricted file should only be accessible if the embargo has ended
-
-        Embargo testEmbargo = new Embargo(LocalDate.now().minusDays(3), "Was embargoed");
-        testDataFile.setEmbargo(testEmbargo);
-
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setRestricted(false);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-
-        assertTrue(sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @ParameterizedTest
-    @CsvSource({"false", "true"})
-    void testCanDownloadFile_forRestrictedReleasedFile(boolean hasPermission) {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setRestricted(true);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-
-        mockPermissionResponseRequestOn(hasPermission);
-
-        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @ParameterizedTest
-    @CsvSource({"false", "true"})
-    void testCanDownloadFile_forRestrictedReleasedFileWithActiveEmbargo(boolean hasPermission) {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        // With an active embargo, a restricted file should have the same access regardless of
-        // embargo state (with an active embargo, there's no way to request permissions,
-        // so the hasPermission=true case primarily applies to the original dataset
-        // creators)
-
-        Embargo testEmbargo = new Embargo(LocalDate.now().plusDays(3), "Still embargoed");
-        testDataFile.setEmbargo(testEmbargo);
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setRestricted(true);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-
-        mockPermissionResponseRequestOn(hasPermission);
-
-        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    @ParameterizedTest
-    @CsvSource({"false", "true"})
-    void testCanDownloadFile_forRestrictedReleasedFileWithExpiredEmbargo(boolean hasPermission) {
-        DataFile testDataFile = new DataFile();
-        testDataFile.setId(2L);
-
-        // With an embargo, a restricted file should have the same access regardless of
-        // embargo state (with an active embargo, there's no way to request permissions,
-        // so the hasPermission=true case primarily applies to the original dataset
-        // creators)
-
-        Embargo testEmbargo = new Embargo(LocalDate.now().minusDays(3), "No longer embargoed");
-        testDataFile.setEmbargo(testEmbargo);
-        DatasetVersion testDatasetVersion = new DatasetVersion();
-        testDatasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
-
-        FileMetadata testFileMetadata = new FileMetadata();
-        testFileMetadata.setId(1L);
-        testFileMetadata.setRestricted(true);
-        testFileMetadata.setDataFile(testDataFile);
-        testFileMetadata.setDatasetVersion(testDatasetVersion);
-
-        mockPermissionResponseRequestOn(hasPermission);
-
-        Assertions.assertEquals(hasPermission, sut.canDownloadFile(testUser, testFileMetadata));
-    }
-
-    private void mockPermissionResponseUserOn(boolean response) {
-        PermissionServiceBean.StaticPermissionQuery staticPermissionQueryMock = mock(PermissionServiceBean.StaticPermissionQuery.class);
-
-        when(permissionServiceBeanStub.userOn(ArgumentMatchers.any(), ArgumentMatchers.any(Dataset.class))).thenReturn(staticPermissionQueryMock);
-        when(staticPermissionQueryMock.has(Permission.EditDataset)).thenReturn(response);
-    }
-
-    private void mockPermissionResponseRequestOn(boolean response) {
-        PermissionServiceBean.RequestPermissionQuery requestPermissionQueryMock = mock(PermissionServiceBean.RequestPermissionQuery.class);
-
-        when(permissionServiceBeanStub.requestOn(ArgumentMatchers.any(), ArgumentMatchers.any(DataFile.class))).thenReturn(requestPermissionQueryMock);
-        when(requestPermissionQueryMock.has(Permission.DownloadFile)).thenReturn(response);
-    }
-}
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index b2f84bd3229..131beeec6b4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2145,38 +2145,6 @@ public void testGetCountGuestbookResponses() throws InterruptedException {
         getCountGuestbookResponsesInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
 
-    @Test
-    public void testCanFileBeDownloaded() {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Upload test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Assert user can download test file
-        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response canFileBeDownloadedResponse = UtilIT.canFileBeDownloaded(Integer.toString(testFileId), apiToken);
-
-        canFileBeDownloadedResponse.then().assertThat().statusCode(OK.getStatusCode());
-        boolean canFileBeDownloaded = JsonPath.from(canFileBeDownloadedResponse.body().asString()).getBoolean("data");
-        assertTrue(canFileBeDownloaded);
-
-        // Call with invalid file id
-        Response canFileBeDownloadedInvalidIdResponse = UtilIT.canFileBeDownloaded("testInvalidId", apiToken);
-        canFileBeDownloadedInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
-    }
-
     @Test
     public void testGetFileThumbnailClass() {
         Response createUser = UtilIT.createRandomUser();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index de3df705714..feaf0635f22 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3284,12 +3284,6 @@ static Response getCountGuestbookResponses(String dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/guestbookResponses/count");
     }
 
-    static Response canFileBeDownloaded(String dataFileId, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/files/" + dataFileId + "/canBeDownloaded");
-    }
-
     static Response getFileThumbnailClass(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)

From b52b33dc58962a8356892c497b186dccf63dd9d3 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 2 Aug 2023 14:29:01 +0100
Subject: [PATCH 264/922] Added: getUserPermissionsOnFile endpoint to Access
 API

---
 .../iq/dataverse/FileDownloadServiceBean.java | 14 ++++++-
 .../edu/harvard/iq/dataverse/api/Access.java  | 23 +++++++++--
 .../harvard/iq/dataverse/api/AccessIT.java    | 40 ++++++++++++++++++-
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 +++
 4 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 18980ff9eff..bb25608033a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -8,6 +8,7 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateGuestbookResponseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RequestAccessCommand;
@@ -18,7 +19,6 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
-import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import java.io.IOException;
@@ -36,7 +36,6 @@
 import javax.inject.Named;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
-import javax.persistence.Query;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServletResponse;
 
@@ -572,4 +571,15 @@ public String getDirectStorageLocatrion(String storageLocation) {
             
         return null; 
     }
+
+    /**
+     *  Checks if the DataverseRequest, which contains IP Groups, has permission to download the file
+     *
+     * @param dataverseRequest the DataverseRequest
+     * @param dataFile the DataFile to check permissions
+     * @return boolean
+     */
+    public boolean canDownloadFile(DataverseRequest dataverseRequest, DataFile dataFile) {
+        return permissionService.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 02441a9ee11..5ea36188ec4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -31,7 +31,7 @@
 import edu.harvard.iq.dataverse.RoleAssignment;
 import edu.harvard.iq.dataverse.UserNotification;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
-import static edu.harvard.iq.dataverse.api.AbstractApiBean.error;
+
 import static edu.harvard.iq.dataverse.api.Datasets.handleVersion;
 
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
@@ -121,8 +121,6 @@
 import javax.ws.rs.core.StreamingOutput;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
 import java.net.URISyntaxException;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import javax.json.JsonObjectBuilder;
 import javax.ws.rs.RedirectionException;
 import javax.ws.rs.ServerErrorException;
@@ -1946,5 +1944,22 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin
             throw new BadRequestException(); 
         }
         return redirectUri;
-    }   
+    }
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userPermissions")
+    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        User requestUser = getRequestUser(crc);
+        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
+        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
+        return ok(jsonObjectBuilder);
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index d6aac80b435..1fdbcab2cc4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -23,8 +23,12 @@
 import java.io.InputStream;
 import java.nio.file.Path;
 import java.util.HashMap;
-import static javax.ws.rs.core.Response.Status.OK;
+
 import org.hamcrest.collection.IsMapContaining;
+
+import javax.json.Json;
+
+import static javax.ws.rs.core.Response.Status.*;
 import static junit.framework.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
@@ -629,7 +633,39 @@ public void testZipUploadAndDownload() throws IOException {
         System.out.println("MD5 checksums of the unzipped file streams are correct.");
         
         System.out.println("Zip upload-and-download round trip test: success!");
-        
     }
 
+    @Test
+    public void testGetUserPermissionsOnFile() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert user permissions on file
+        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(testFileId), apiToken);
+
+        getUserPermissionsOnFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean canDownloadFile = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canDownloadFile");
+        assertTrue(canDownloadFile);
+        boolean canEditOwnerDataset = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canEditOwnerDataset");
+        assertTrue(canEditOwnerDataset);
+
+        // Call with invalid file id
+        Response getUserPermissionsOnFileInvalidIdResponse = UtilIT.getUserPermissionsOnFile("testInvalidId", apiToken);
+        getUserPermissionsOnFileInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index feaf0635f22..f39e246863d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3295,4 +3295,10 @@ static Response getFileDataTables(String dataFileId, String apiToken) {
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/dataTables");
     }
+
+    static Response getUserPermissionsOnFile(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/access/datafile/" + dataFileId + "/userPermissions");
+    }
 }

From 1f145f8f3c8d69570e21c97d8ca89d875158e880 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 2 Aug 2023 09:42:13 -0400
Subject: [PATCH 265/922] enable html in emails

---
 .../edu/harvard/iq/dataverse/FileDownloadServiceBean.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index dcfe58c5830..ec35c214a22 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -571,10 +571,10 @@ public boolean requestAccess(DataFile dataFile, GuestbookResponse gbr){
     public void sendRequestFileAccessNotification(Dataset dataset, Long fileId, AuthenticatedUser requestor) {
         Timestamp ts = new Timestamp(new Date().getTime());
         permissionService.getUsersWithPermissionOn(Permission.ManageDatasetPermissions, dataset).stream().forEach((au) -> {
-            userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, false);
+            userNotificationService.sendNotification(au, ts, UserNotification.Type.REQUESTFILEACCESS, fileId, null, requestor, true);
         });
         //send the user that requested access a notification that they requested the access
-        userNotificationService.sendNotification(requestor, ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId, null, requestor, false);
+        userNotificationService.sendNotification(requestor, ts, UserNotification.Type.REQUESTEDFILEACCESS, fileId, null, requestor, true);
 
     } 
     

From 36d26d4b0ef9185869a006d78ca3be371dc19112 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 2 Aug 2023 11:52:39 -0400
Subject: [PATCH 266/922] update test cred

---
 .../harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 0d7c5458e14..081c5a622aa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -691,7 +691,7 @@ public static void main(String[] args) {
         }
         System.setProperty("dataverse.files.globus.base-uri", "2791b83e-b989-47c5-a7fa-ce65fd949522");
         System.out.println("Valid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
-        System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOlprRmxGejNTWDlkTVpUNk92ZmVJaFQyTWY0SDd4cXBoTDNSS29vUmRGVlE9");
+        System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOmtsa1RZc242bU1oRXNuUFFwQy9oSzQxSi9EMDV6SjRtUDd1c0ZiN011MEk9");
         System.setProperty("dataverse.files.globus.base-store","file");
         System.setProperty("dataverse.files.file.type",
                 DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);

From fdc9bee1b9bb6a8285771e257e249778e201e32f Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 2 Aug 2023 13:32:06 -0400
Subject: [PATCH 267/922] fix html message formatting

---
 .../iq/dataverse/GuestbookResponse.java       | 25 ++++++++++---------
 .../harvard/iq/dataverse/MailServiceBean.java |  8 +++---
 src/main/java/propertyFiles/Bundle.properties |  2 +-
 3 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 60205eeabcf..e274d3b130b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -312,27 +312,28 @@ public void setSessionId(String sessionId) {
         this.fileDownload.setSessionId(sessionId);
     }
     
-    public String toFormattedResponse() {
+    public String toHtmlFormattedResponse() {
 
         StringBuilder sb = new StringBuilder();
 
-        sb.append("\n" + BundleUtil.getStringFromBundle("dataset.guestbookResponse.id") + ": " + getId() + "\n");
-        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.date") + ": " + getResponseDate() + "\n");
-        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.respondent") + "\n  "
-                + BundleUtil.getStringFromBundle("name") + ": " + getName() + "\n");
-        sb.append("  " + BundleUtil.getStringFromBundle("email") + ": " + getEmail() + "\n");
+        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.id") + ": " + getId() + "<br>\n");
+        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.date") + ": " + getResponseDate() + "<br>\n");
+        sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.respondent") + "<br><ul style=\"list-style-type:none;\">\n<li>"
+                + BundleUtil.getStringFromBundle("name") + ": " + getName() + "</li>\n<li>");
+        sb.append("  " + BundleUtil.getStringFromBundle("email") + ": " + getEmail() + "</li>\n<li>");
         sb.append(
-                "  " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "\n");
-        sb.append("  " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "\n");
+                "  " + BundleUtil.getStringFromBundle("institution") + ": " + wrapNullAnswer(getInstitution()) + "</li>\n<li>");
+        sb.append("  " + BundleUtil.getStringFromBundle("position") + ": " + wrapNullAnswer(getPosition()) + "</li></ul>\n");
         sb.append(BundleUtil.getStringFromBundle("dataset.guestbookResponse.guestbook.additionalQuestions")
-                + ":\n");
+                + ":<ul style=\"list-style-type:none;\">\n");
 
         for (CustomQuestionResponse cqr : getCustomQuestionResponses()) {
-            sb.append("  " + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": "
-                    + cqr.getCustomQuestion().getQuestionString() + "\n  "
+            sb.append("<li>" + BundleUtil.getStringFromBundle("dataset.guestbookResponse.question") + ": "
+                    + cqr.getCustomQuestion().getQuestionString() + "<br>"
                     + BundleUtil.getStringFromBundle("dataset.guestbookResponse.answer") + ": "
-                    + wrapNullAnswer(cqr.getResponse()) + "\n\n");
+                    + wrapNullAnswer(cqr.getResponse()) + "</li>\n");
         }
+        sb.append("</ul>");
         return sb.toString();
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
index 5b69641f029..3db547cf305 100644
--- a/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/MailServiceBean.java
@@ -394,12 +394,13 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                 String requestorName = (requestor.getLastName() != null && requestor.getLastName() != null) ? requestor.getFirstName() + " " + requestor.getLastName() : BundleUtil.getStringFromBundle("notification.email.info.unavailable");
                 String requestorEmail = requestor.getEmail() != null ? requestor.getEmail() : BundleUtil.getStringFromBundle("notification.email.info.unavailable"); 
                 String[] paramArrayRequestFileAccess = {datafile.getOwner().getDisplayName(), requestorName, requestorEmail, getDatasetManageFileAccessLink(datafile)};
+                messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html");
                 messageText += MessageFormat.format(pattern, paramArrayRequestFileAccess);
                 FileAccessRequest far = datafile.getAccessRequestForAssignee(requestor);
                 GuestbookResponse gbr = far.getGuestbookResponse();
                 if (gbr != null) {
                     messageText += MessageFormat.format(
-                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toFormattedResponse());
+                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toHtmlFormattedResponse());
                 }
                 return messageText;
             case GRANTFILEACCESS:
@@ -648,12 +649,13 @@ public String getMessageTextBasedOnNotification(UserNotification userNotificatio
                 datafile = (DataFile) targetObject;
                 
                 pattern = BundleUtil.getStringFromBundle("notification.email.requestedFileAccess");
-                messageText += MessageFormat.format(pattern, getDvObjectLink(datafile), datafile.getOwner().getDisplayName());
+                 messageText = BundleUtil.getStringFromBundle("notification.email.greeting.html");
+                 messageText += MessageFormat.format(pattern, getDvObjectLink(datafile), datafile.getOwner().getDisplayName());
                 far = datafile.getAccessRequestForAssignee(requestor);
                 gbr = far.getGuestbookResponse();
                 if (gbr != null) {
                     messageText += MessageFormat.format(
-                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toFormattedResponse());
+                            BundleUtil.getStringFromBundle("notification.email.requestFileAccess.guestbookResponse"), gbr.toHtmlFormattedResponse());
                 }
                 return messageText;
         }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index edc3c387af4..8c5e3ec6aa2 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -772,7 +772,7 @@ notification.email.greeting.html=Hello, <br>
 notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance.
 notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0} . Note, the verify link will expire after {1}. Send another verification email by visiting your account page.
 notification.email.requestFileAccess=File access requested for dataset: {0} by {1} ({2}). Manage permissions at {3}.
-notification.email.requestFileAccess.guestbookResponse=\n\nGuestbook Response:\n\n {0}
+notification.email.requestFileAccess.guestbookResponse=<br><br>Guestbook Response:<br><br>{0}
 notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1} ).
 notification.email.rejectFileAccess=Your request for access was rejected for the requested files in the dataset: {0} (view at {1} ). If you have any questions about why your request was rejected, you may reach the dataset owner using the "Contact" link on the upper right corner of the dataset page.
 # Bundle file editors, please note that "notification.email.createDataverse" is used in a unit test

From d16264d4d7c0f68c310b61189288b3021421ee05 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 4 Aug 2023 10:36:54 +0100
Subject: [PATCH 268/922] Removed: getFileThumbnailClass API endpoint

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 12 -------
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 32 -------------------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 ----
 3 files changed, 50 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 83c464f2a27..3ea7a6336b0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -109,8 +109,6 @@ public class Files extends AbstractApiBean {
     MakeDataCountLoggingServiceBean mdcLogService;
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
-    @Inject
-    DataFileServiceBean dataFileServiceBean;
 
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
@@ -838,16 +836,6 @@ public Response getCountGuestbookResponses(@Context ContainerRequestContext crc,
         }, getRequestUser(crc));
     }
 
-    @GET
-    @AuthRequired
-    @Path("{id}/thumbnailClass")
-    public Response getFileThumbnailClass(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        return response(req -> {
-            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
-            return ok(dataFileServiceBean.getFileThumbnailClass(dataFile));
-        }, getRequestUser(crc));
-    }
-
     @GET
     @AuthRequired
     @Path("{id}/dataTables")
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 131beeec6b4..e1cd6128fc0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2145,38 +2145,6 @@ public void testGetCountGuestbookResponses() throws InterruptedException {
         getCountGuestbookResponsesInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
 
-    @Test
-    public void testGetFileThumbnailClass() {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Upload test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Get file thumbnail class and assert is image
-        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response getFileThumbnailClassResponse = UtilIT.getFileThumbnailClass(Integer.toString(testFileId), apiToken);
-
-        getFileThumbnailClassResponse.then().assertThat().statusCode(OK.getStatusCode());
-        String fileThumbnailClass = JsonPath.from(getFileThumbnailClassResponse.body().asString()).getString("data.message");
-        assertEquals("image", fileThumbnailClass);
-
-        // Call with invalid file id
-        Response getFileThumbnailClassInvalidIdResponse = UtilIT.getFileThumbnailClass("testInvalidId", apiToken);
-        getFileThumbnailClassInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
-    }
-
     @Test
     public void testGetFileDataTables() throws InterruptedException {
         Response createUser = UtilIT.createRandomUser();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index f39e246863d..603e3be12cd 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3284,12 +3284,6 @@ static Response getCountGuestbookResponses(String dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/guestbookResponses/count");
     }
 
-    static Response getFileThumbnailClass(String dataFileId, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/files/" + dataFileId + "/thumbnailClass");
-    }
-
     static Response getFileDataTables(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)

From 0952a62ebb47a1ae5d6fd14d64557e55fe82f45c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 4 Aug 2023 11:11:58 +0100
Subject: [PATCH 269/922] Changed: release notes

---
 doc/release-notes/9692-files-api-extension.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/doc/release-notes/9692-files-api-extension.md b/doc/release-notes/9692-files-api-extension.md
index 03484083113..2dc441e7250 100644
--- a/doc/release-notes/9692-files-api-extension.md
+++ b/doc/release-notes/9692-files-api-extension.md
@@ -1,8 +1,7 @@
 The following API endpoints have been added:
 
 - /api/files/{id}/guestbookResponses/count
-- /api/files/{id}/canBeDownloaded
-- /api/files/{id}/thumbnailClass
 - /api/files/{id}/dataTables
+- /access/datafile/{id}/userPermissions
 
 The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination and ordering

From 02cecd4666c3093167f55245774a5f54e4dadf13 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 4 Aug 2023 11:12:49 +0100
Subject: [PATCH 270/922] Added: docs for files endpoint pagination and order
 criteria

---
 doc/sphinx-guides/source/api/native-api.rst | 23 +++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 2bbcf108cad..8dad8ab13a9 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -958,6 +958,29 @@ The fully expanded example above (without environment variables) looks like this
  
   curl https://demo.dataverse.org/api/datasets/24/versions/1.0/files
 
+This endpoint supports optional pagination, through the ``limit`` and ``offset`` query params:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
+
+Ordering criteria for sorting the results, like in the Dataverse UI, is also optionally supported. In particular, by the following possible values:
+
+* ``NameAZ`` (Default)
+* ``NameZA``
+* ``Newest``
+* ``Oldest``
+* ``Size``
+* ``Type``
+
+Please note that these values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?orderCriteria=Newest"
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 6722777096fdf0262b7663add39b360277276d2d Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 4 Aug 2023 11:21:38 +0100
Subject: [PATCH 271/922] Added: docs for file dataTables API endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 38 +++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 8dad8ab13a9..4b96efdb3ba 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2725,6 +2725,44 @@ The fully expanded example above (without environment variables) looks like this
 
 Note: The ``id`` returned in the json response is the id of the file metadata version.
 
+Getting File Data Tables
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This endpoint is oriented to tabular files, and provides a json representation of the file data tables for an existing tabular file. ``ID`` is the database id of the file to get the data tables from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl $SERVER_URL/api/files/$ID/dataTables
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl https://demo.dataverse.org/api/files/24/dataTables
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl "$SERVER_URL/api/files/:persistentId/dataTables?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/files/:persistentId/dataTables?persistentId=doi:10.5072/FK2/AAA000"
+
+Note that if the requested file is not tabular, the endpoint will return an error.
 
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~

From 90ef2f11e8e4a1dffe3bc1f632b7c55afed6bbd0 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 4 Aug 2023 11:28:47 +0100
Subject: [PATCH 272/922] Added: docs for guestbookResponses/count files API
 endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 37 +++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 4b96efdb3ba..2bd8e3a02d6 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2764,6 +2764,43 @@ The fully expanded example above (without environment variables) looks like this
 
 Note that if the requested file is not tabular, the endpoint will return an error.
 
+Getting File Guestbook Response Count
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Provides the guestbook response count for a particular file, where ``ID`` is the database id of the file to get the count from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl $SERVER_URL/api/files/$ID/guestbookResponses/count
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl https://demo.dataverse.org/api/files/24/guestbookResponses/count
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl "$SERVER_URL/api/files/:persistentId/guestbookResponses/count?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/files/:persistentId/guestbookResponses/count?persistentId=doi:10.5072/FK2/AAA000"
+
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 

From a7b2485b921b9acdba6287faca8c5e5ca2e438e1 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 4 Aug 2023 11:30:55 +0100
Subject: [PATCH 273/922] Changed: version files endpoint docs tweak

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 2bd8e3a02d6..1d8b4c987c2 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -964,7 +964,7 @@ This endpoint supports optional pagination, through the ``limit`` and ``offset``
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
 
-Ordering criteria for sorting the results, like in the Dataverse UI, is also optionally supported. In particular, by the following possible values:
+Ordering criteria for sorting the results is also optionally supported. In particular, by the following possible values:
 
 * ``NameAZ`` (Default)
 * ``NameZA``

From cd8e2292f19b6e2f7c64fd34826a6e090428bd81 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 4 Aug 2023 10:44:12 -0400
Subject: [PATCH 274/922] wording #9692

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 1d8b4c987c2..e14ed3a90c7 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2728,7 +2728,7 @@ Note: The ``id`` returned in the json response is the id of the file metadata ve
 Getting File Data Tables
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-This endpoint is oriented to tabular files, and provides a json representation of the file data tables for an existing tabular file. ``ID`` is the database id of the file to get the data tables from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
+This endpoint is oriented toward tabular files and provides a JSON representation of the file data tables for an existing tabular file. ``ID`` is the database id of the file to get the data tables from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
 
 A curl example using an ``ID``
 

From 369f1a8f5c261219c0f153471c26f3f35b88b10f Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 8 Aug 2023 13:53:35 -0400
Subject: [PATCH 275/922] Fix popup with no ToA

---
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 755e9e4b814..354c2d5b9fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1603,7 +1603,11 @@ public static boolean isRequestAccessPopupRequired(DatasetVersion datasetVersion
         if (answer != null) {
             return answer;
         }
-        //ToDo - also check for guestbook when guestbook at request is true?
+     // 3. Guest Book:
+        if (datasetVersion.getDataset() != null && datasetVersion.getDataset().getGuestbook() != null && datasetVersion.getDataset().getGuestbook().isEnabled() && datasetVersion.getDataset().getGuestbook().getDataverse() != null) {
+            logger.fine("Request access popup required because of guestbook.");
+            return true;
+        }
         logger.fine("Request access popup is not required.");
         return false;
     }

From e30429bd2e9860658b19913c26d171d264255f38 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 8 Aug 2023 13:53:52 -0400
Subject: [PATCH 276/922] fix validation/error messages

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 8b89bb6107d..69cc9fae55c 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -142,6 +142,9 @@
                 escape="false" />
         </div>
             <p:fragment rendered="#{guestbookAndTermsPopupRequired and ((termsGuestbookPopupAction != bundle.download) != guestbookPopupRequiredAtDownload)}" id="guestbookUIFragment">
+                <div class="container messagePanel">
+                    <iqbs:messages collapsible="true" />
+                </div>
                 <!--
                 Only validate the active context. For example, the user might be filling out
                 the guestbook on the Preview tab rather than after clicking the "download file"
@@ -304,16 +307,16 @@
         <!--TODO - add "Accept and" if ToA exist?-->
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{!guestbookPopupRequiredAtDownload}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.writeGuestbookResponseAndRequestAccess(guestbookResponse)}"
-                    update="@([id$=messagePanel])"
+                    update="guestbookUIFragment"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
-                    <f:param name="DO_GB_VALIDATION" value="true"/>
+                    <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
             </p:commandLink>
 
             <p:commandLink type="button" styleClass="btn btn-default" rendered="#{guestbookPopupRequiredAtDownload}" value="#{bundle['file.requestAccess']}"
                     actionListener="#{fileDownloadHelper.requestAccessIndirect()}"
-                    update="@([id$=messagePanel])"
+                    update="guestbookUIFragment"
                     oncomplete="if (!args || (args &amp;&amp; !args.validationFailed)){PF('guestbookAndTermsPopup').hide();window.location.replace(window.location.href);}">
-                    <f:param name="DO_GB_VALIDATION" value="true"/>
+                    <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
             </p:commandLink>
 
             <button type="button" class="btn btn-default" onclick="PF('guestbookAndTermsPopup').hide();PF('blockDatasetForm').hide();" value="#{bundle.cancel}">#{bundle.cancel}</button>

From a64199fbda39c34215ae368b90f36a487ebcc651 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 9 Aug 2023 08:21:44 +0100
Subject: [PATCH 277/922] Added: API docs for access file user permissions

---
 doc/sphinx-guides/source/api/dataaccess.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index e76ea167587..2e5c368be47 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -403,3 +403,19 @@ This method returns a list of Authenticated Users who have requested access to t
 A curl example using an ``id``::
 
     curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests
+
+Get user permissions on a file:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``/api/access/datafile/{id}/userPermissions``
+
+This method returns the permissions that the calling user has on a particular file.
+
+In particular, the user permissions that this method checks, returned as booleans, are the following:
+
+* Can download the file
+* Can edit the file owner dataset
+
+A curl example using an ``id``::
+
+    curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/userPermissions

From 719fc67b0b93dc1f9a8fdc38e44dc1f7287f3944 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 9 Aug 2023 09:20:21 +0100
Subject: [PATCH 278/922] Changed: guestbookResponses/count endpoint renamed to
 downloadCount

---
 doc/release-notes/9692-files-api-extension.md      |  2 +-
 doc/sphinx-guides/source/api/native-api.rst        | 14 +++++++-------
 .../java/edu/harvard/iq/dataverse/api/Files.java   |  4 ++--
 .../java/edu/harvard/iq/dataverse/api/FilesIT.java | 14 +++++++-------
 .../java/edu/harvard/iq/dataverse/api/UtilIT.java  |  4 ++--
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/doc/release-notes/9692-files-api-extension.md b/doc/release-notes/9692-files-api-extension.md
index 2dc441e7250..baa8e2f87cd 100644
--- a/doc/release-notes/9692-files-api-extension.md
+++ b/doc/release-notes/9692-files-api-extension.md
@@ -1,6 +1,6 @@
 The following API endpoints have been added:
 
-- /api/files/{id}/guestbookResponses/count
+- /api/files/{id}/downloadCount
 - /api/files/{id}/dataTables
 - /access/datafile/{id}/userPermissions
 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index e14ed3a90c7..34a6717ada2 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2764,10 +2764,10 @@ The fully expanded example above (without environment variables) looks like this
 
 Note that if the requested file is not tabular, the endpoint will return an error.
 
-Getting File Guestbook Response Count
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Getting File Download Count
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Provides the guestbook response count for a particular file, where ``ID`` is the database id of the file to get the count from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
+Provides the download count for a particular file, where ``ID`` is the database id of the file to get the download count from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
 
 A curl example using an ``ID``
 
@@ -2777,13 +2777,13 @@ A curl example using an ``ID``
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
 
-  curl $SERVER_URL/api/files/$ID/guestbookResponses/count
+  curl $SERVER_URL/api/files/$ID/downloadCount
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl https://demo.dataverse.org/api/files/24/guestbookResponses/count
+  curl https://demo.dataverse.org/api/files/24/downloadCount
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -2793,13 +2793,13 @@ A curl example using a ``PERSISTENT_ID``
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
-  curl "$SERVER_URL/api/files/:persistentId/guestbookResponses/count?persistentId=$PERSISTENT_ID"
+  curl "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/:persistentId/guestbookResponses/count?persistentId=doi:10.5072/FK2/AAA000"
+  curl "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
 
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 3ea7a6336b0..d065c764bbe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -828,8 +828,8 @@ public Response getFixityAlgorithm() {
 
     @GET
     @AuthRequired
-    @Path("{id}/guestbookResponses/count")
-    public Response getCountGuestbookResponses(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+    @Path("{id}/downloadCount")
+    public Response getFileDownloadCount(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
         return response(req -> {
             DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
             return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index e1cd6128fc0..98fc6b55150 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2103,7 +2103,7 @@ public void testFilePIDsBehavior() {
     }
 
     @Test
-    public void testGetCountGuestbookResponses() throws InterruptedException {
+    public void testGetFileDownloadCount() throws InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -2131,18 +2131,18 @@ public void testGetCountGuestbookResponses() throws InterruptedException {
         Response downloadResponse = UtilIT.downloadFile(testFileId, apiToken);
         downloadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Ensure guestbook is updated
+        // Ensure download count is updated
         sleep(2000);
 
-        // Get count guestbook responses and assert it is 1
-        Response getCountGuestbookResponsesResponse = UtilIT.getCountGuestbookResponses(Integer.toString(testFileId), apiToken);
-        getCountGuestbookResponsesResponse.then().assertThat()
+        // Get download count and assert it is 1
+        Response getFileDownloadCountResponse = UtilIT.getFileDownloadCount(Integer.toString(testFileId), apiToken);
+        getFileDownloadCountResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data.message", equalTo("1"));
 
         // Call with invalid file id
-        Response getCountGuestbookResponsesInvalidIdResponse = UtilIT.getCountGuestbookResponses("testInvalidId", apiToken);
-        getCountGuestbookResponsesInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+        Response getFileDownloadCountInvalidIdResponse = UtilIT.getFileDownloadCount("testInvalidId", apiToken);
+        getFileDownloadCountInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 603e3be12cd..2151a68f7da 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3278,10 +3278,10 @@ static Response createAndUploadTestFile(String persistentId, String testFileName
         return uploadZipFileViaSword(persistentId, pathToTestFile, apiToken);
     }
 
-    static Response getCountGuestbookResponses(String dataFileId, String apiToken) {
+    static Response getFileDownloadCount(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/files/" + dataFileId + "/guestbookResponses/count");
+                .get("/api/files/" + dataFileId + "/downloadCount");
     }
 
     static Response getFileDataTables(String dataFileId, String apiToken) {

From 12243115d4e7effce29338a8dcfd8f4f7c111458 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 9 Aug 2023 10:07:01 -0400
Subject: [PATCH 279/922] doc tweaks #9692

---
 doc/sphinx-guides/source/api/dataaccess.rst | 4 ++--
 doc/sphinx-guides/source/api/metrics.rst    | 7 +++++++
 doc/sphinx-guides/source/api/native-api.rst | 8 +++++---
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 2e5c368be47..d714c90372a 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -404,7 +404,7 @@ A curl example using an ``id``::
 
     curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests
 
-Get user permissions on a file:
+Get User Permissions on a File:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 ``/api/access/datafile/{id}/userPermissions``
@@ -418,4 +418,4 @@ In particular, the user permissions that this method checks, returned as boolean
 
 A curl example using an ``id``::
 
-    curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/userPermissions
+    curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userPermissions"
diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst
index 28ac33ea228..613671e49d1 100755
--- a/doc/sphinx-guides/source/api/metrics.rst
+++ b/doc/sphinx-guides/source/api/metrics.rst
@@ -163,3 +163,10 @@ The following table lists the available metrics endpoints (not including the Mak
     /api/info/metrics/uniquefiledownloads/toMonth/{yyyy-MM},"count by id, pid","json, csv",collection subtree,published,y,cumulative up to month specified,unique download counts per file id to the specified month. PIDs are also included in output if they exist
     /api/info/metrics/tree,"id, ownerId, alias, depth, name, children",json,collection subtree,published,y,"tree of dataverses starting at the root or a specified parentAlias with their id, owner id, alias, name, a computed depth, and array of children dataverses","underlying code can also include draft dataverses, this is not currently accessible via api, depth starts at 0"
     /api/info/metrics/tree/toMonth/{yyyy-MM},"id, ownerId, alias, depth, name, children",json,collection subtree,published,y,"tree of dataverses in existence as of specified date starting at the root or a specified parentAlias with their id, owner id, alias, name, a computed depth, and array of children dataverses","underlying code can also include draft dataverses, this is not currently accessible via api, depth starts at 0"
+
+Related API Endpoints
+---------------------
+
+The following endpoints are not under the metrics namespace but also return counts:
+
+- :ref:`file-download-count` 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 34a6717ada2..236448de19c 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -956,7 +956,7 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
  
-  curl https://demo.dataverse.org/api/datasets/24/versions/1.0/files
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files"
 
 This endpoint supports optional pagination, through the ``limit`` and ``offset`` query params:
 
@@ -2764,6 +2764,8 @@ The fully expanded example above (without environment variables) looks like this
 
 Note that if the requested file is not tabular, the endpoint will return an error.
 
+.. _file-download-count:
+
 Getting File Download Count
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -2777,13 +2779,13 @@ A curl example using an ``ID``
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
 
-  curl $SERVER_URL/api/files/$ID/downloadCount
+  curl "$SERVER_URL/api/files/$ID/downloadCount"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl https://demo.dataverse.org/api/files/24/downloadCount
+  curl "https://demo.dataverse.org/api/files/24/downloadCount"
 
 A curl example using a ``PERSISTENT_ID``
 

From 7db5bcb9fff919c846f8494cd1f7fae98db0ddee Mon Sep 17 00:00:00 2001
From: sferey <steven.ferey@smile.fr>
Date: Wed, 9 Aug 2023 17:20:13 +0200
Subject: [PATCH 280/922] exclusion of ingested files during mimetype
 redetection

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index f6eda085c95..c915e4be5ec 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -726,6 +726,11 @@ public Response reingest(@Context ContainerRequestContext crc, @PathParam("id")
     public Response redetectDatafile(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("dryRun") boolean dryRun) {
         try {
             DataFile dataFileIn = findDataFileOrDie(id);
+            // Ingested Files have mimetype = text/tab-separated-values
+            // No need to redetect
+            if (dataFileIn.isTabularData()) {
+                return error(Response.Status.BAD_REQUEST, "The file is an ingested tabular file.");
+            }
             String originalContentType = dataFileIn.getContentType();
             DataFile dataFileOut = execCommand(new RedetectFileTypeCommand(createDataverseRequest(getRequestUser(crc)), dataFileIn, dryRun));
             NullSafeJsonBuilder result = NullSafeJsonBuilder.jsonObjectBuilder()

From ebb9c9a107c277972fe1e3fa1140735991ebbfe2 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Aug 2023 08:33:20 +0100
Subject: [PATCH 281/922] Added: QueryDSL setup and used for querying dataset
 version files (pending filters)

---
 modules/dataverse-parent/pom.xml              |  3 +
 pom.xml                                       | 28 ++++++
 .../dataverse/DatasetVersionServiceBean.java  | 85 +++++++++++--------
 .../harvard/iq/dataverse/api/Datasets.java    | 14 ++-
 4 files changed, 93 insertions(+), 37 deletions(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 7d46f7c49b9..246e80bf060 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -200,6 +200,9 @@
         
         <!-- Container related -->
         <fabric8-dmp.version>0.43.0</fabric8-dmp.version>
+
+        <!-- Persistence -->
+        <querydsl.version>5.0.0</querydsl.version>
     </properties>
     
     <pluginRepositories>
diff --git a/pom.xml b/pom.xml
index 96f598af0f5..a0fb04abea9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -230,6 +230,18 @@
             <artifactId>jakarta.el</artifactId>
             <scope>provided</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.querydsl</groupId>
+            <artifactId>querydsl-apt</artifactId>
+            <version>${querydsl.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.querydsl</groupId>
+            <artifactId>querydsl-jpa</artifactId>
+            <version>${querydsl.version}</version>
+        </dependency>
         
         <dependency>
             <groupId>commons-io</groupId>
@@ -750,6 +762,22 @@
                     <consoleOutput>true</consoleOutput>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>com.mysema.maven</groupId>
+                <artifactId>apt-maven-plugin</artifactId>
+                <version>1.1.3</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>process</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>target/generated-sources/java</outputDirectory>
+                            <processor>com.querydsl.apt.jpa.JPAAnnotationProcessor</processor>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
     <profiles>
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index a57a8bf39c2..07ae2db0cbd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -1,5 +1,9 @@
 package edu.harvard.iq.dataverse;
 
+import com.querydsl.core.types.dsl.CaseBuilder;
+import com.querydsl.core.types.dsl.DateTimeExpression;
+import com.querydsl.jpa.impl.JPAQuery;
+import com.querydsl.jpa.impl.JPAQueryFactory;
 import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
 import edu.harvard.iq.dataverse.ingest.IngestUtil;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
@@ -8,12 +12,15 @@
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import static edu.harvard.iq.dataverse.batch.jobs.importer.filesystem.FileRecordJobListener.SEP;
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
+import edu.harvard.iq.dataverse.QDvObject;
+import edu.harvard.iq.dataverse.QFileMetadata;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.MarkupChecker;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.IOException;
+import java.sql.Timestamp;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -37,7 +44,7 @@
 import javax.persistence.TypedQuery;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.solr.client.solrj.SolrServerException;
-    
+
 /**
  *
  * @author skraffmiller
@@ -50,22 +57,6 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
-            + " WHERE fm.datasetVersion.id=:datasetVersionId"
-            + " ORDER BY fm.label";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = dvo.id"
-            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = df.id"
-            + " ORDER BY df.filesize";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = df.id"
-            + " ORDER BY df.contentType";
-
     @EJB
     DatasetServiceBean datasetService;
     
@@ -1258,45 +1249,69 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
      * Returns a FileMetadata list of files in the specified DatasetVersion
      *
      * @param datasetVersion the DatasetVersion to access
-     * @param limit for pagination, can be null
-     * @param offset for pagination, can be null
-     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
+     * @param limit          for pagination, can be null
+     * @param offset         for pagination, can be null
+     * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
      * @return a FileMetadata list of the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
-        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
-                .setParameter("datasetVersionId", datasetVersion.getId());
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion,
+                                               Integer limit,
+                                               Integer offset,
+                                               String fileType,
+                                               String fileAccess,
+                                               String fileTag,
+                                               FileMetadatasOrderCriteria orderCriteria) {
+        JPAQuery<FileMetadata> query = createQueryFromCriteria(datasetVersion, orderCriteria);
         if (limit != null) {
-            query.setMaxResults(limit);
+            query.limit(limit);
         }
         if (offset != null) {
-            query.setFirstResult(offset);
+            query.offset(offset);
         }
-        return query.getResultList();
+        return query.fetch();
     }
 
-    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
-        String queryString;
+    // TODO: Refactor
+    private JPAQuery<FileMetadata> createQueryFromCriteria(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
+        QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
+        QDvObject dvObject = QDvObject.dvObject;
+        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        JPAQuery<FileMetadata> query;
         switch (orderCriteria) {
             case NameZA:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
+                query = queryFactory.selectFrom(fileMetadata)
+                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                        .orderBy(fileMetadata.label.desc());
                 break;
             case Newest:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
+                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject)
+                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                        .where(dvObject.id.eq(fileMetadata.dataFile.id))
+                        .orderBy(orderByLifetimeExpression.desc());
                 break;
             case Oldest:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
+                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject)
+                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                        .where(dvObject.id.eq(fileMetadata.dataFile.id))
+                        .orderBy(orderByLifetimeExpression.asc());
                 break;
             case Size:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
+                query = queryFactory.selectFrom(fileMetadata)
+                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                        .orderBy(fileMetadata.dataFile.filesize.asc());
                 break;
             case Type:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
+                query = queryFactory.selectFrom(fileMetadata)
+                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                        .orderBy(fileMetadata.dataFile.contentType.asc());
                 break;
             default:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
+                query = queryFactory.selectFrom(fileMetadata)
+                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                        .orderBy(fileMetadata.label.asc());
                 break;
         }
-        return queryString;
+        return query;
     }
 } // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 623f4798a5b..4de6e410b82 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -493,7 +493,17 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
-    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+    public Response getVersionFiles(@Context ContainerRequestContext crc,
+                                    @PathParam("id") String datasetId,
+                                    @PathParam("versionId") String versionId,
+                                    @QueryParam("limit") Integer limit,
+                                    @QueryParam("offset") Integer offset,
+                                    @QueryParam("fileType") String fileType,
+                                    @QueryParam("fileAccess") String fileAccess,
+                                    @QueryParam("fileTag") String fileTag,
+                                    @QueryParam("orderCriteria") String orderCriteria,
+                                    @Context UriInfo uriInfo,
+                                    @Context HttpHeaders headers) {
         return response( req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
@@ -502,7 +512,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileType, fileAccess, fileTag, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     

From 83dc3534cad46105a3d0d0f6a1b38083b3c7830e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Aug 2023 08:53:09 +0100
Subject: [PATCH 282/922] Refactor: DatasetVersionFilesServiceBean for dataset
 version file operations

---
 .../DatasetVersionFilesServiceBean.java       |  86 +++++++++++++
 .../dataverse/DatasetVersionServiceBean.java  | 118 ++----------------
 .../harvard/iq/dataverse/api/Datasets.java    |   9 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  |  11 +-
 4 files changed, 109 insertions(+), 115 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
new file mode 100644
index 00000000000..70340e7cb29
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -0,0 +1,86 @@
+package edu.harvard.iq.dataverse;
+
+import edu.harvard.iq.dataverse.QDvObject;
+import edu.harvard.iq.dataverse.QFileMetadata;
+
+import com.querydsl.core.types.dsl.CaseBuilder;
+import com.querydsl.core.types.dsl.DateTimeExpression;
+import com.querydsl.jpa.impl.JPAQuery;
+import com.querydsl.jpa.impl.JPAQueryFactory;
+
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.util.List;
+import javax.ejb.Stateless;
+import javax.inject.Named;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+
+/**
+ * @author Guillermo Portas
+ * TODO
+ */
+@Stateless
+@Named
+public class DatasetVersionFilesServiceBean implements Serializable {
+
+    @PersistenceContext(unitName = "VDCNet-ejbPU")
+    private EntityManager em;
+
+    /**
+     * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
+     */
+    public enum FileMetadatasOrderCriteria {
+        NameAZ, NameZA, Newest, Oldest, Size, Type
+    }
+
+    /**
+     * Returns a FileMetadata list of files in the specified DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param limit          for pagination, can be null
+     * @param offset         for pagination, can be null
+     * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
+     * @return a FileMetadata list of the specified DatasetVersion
+     */
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String fileType, String fileAccess, String fileTag, FileMetadatasOrderCriteria orderCriteria) {
+        JPAQuery<FileMetadata> query = createQueryFromCriteria(datasetVersion, orderCriteria);
+        if (limit != null) {
+            query.limit(limit);
+        }
+        if (offset != null) {
+            query.offset(offset);
+        }
+        return query.fetch();
+    }
+
+    // TODO: Refactor
+    private JPAQuery<FileMetadata> createQueryFromCriteria(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
+        QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
+        QDvObject dvObject = QDvObject.dvObject;
+        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        JPAQuery<FileMetadata> query;
+        switch (orderCriteria) {
+            case NameZA:
+                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.label.desc());
+                break;
+            case Newest:
+                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id)).orderBy(orderByLifetimeExpression.desc());
+                break;
+            case Oldest:
+                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id)).orderBy(orderByLifetimeExpression.asc());
+                break;
+            case Size:
+                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.dataFile.filesize.asc());
+                break;
+            case Type:
+                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.dataFile.contentType.asc());
+                break;
+            default:
+                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.label.asc());
+                break;
+        }
+        return query;
+    }
+} // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 07ae2db0cbd..b5f95bdc00a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -1,37 +1,19 @@
 package edu.harvard.iq.dataverse;
 
-import com.querydsl.core.types.dsl.CaseBuilder;
-import com.querydsl.core.types.dsl.DateTimeExpression;
-import com.querydsl.jpa.impl.JPAQuery;
-import com.querydsl.jpa.impl.JPAQueryFactory;
 import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
-import edu.harvard.iq.dataverse.ingest.IngestUtil;
-import edu.harvard.iq.dataverse.pidproviders.PidUtil;
-import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-import static edu.harvard.iq.dataverse.batch.jobs.importer.filesystem.FileRecordJobListener.SEP;
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
-import edu.harvard.iq.dataverse.QDvObject;
-import edu.harvard.iq.dataverse.QFileMetadata;
+import edu.harvard.iq.dataverse.ingest.IngestUtil;
+import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.MarkupChecker;
 import edu.harvard.iq.dataverse.util.SystemConfig;
-import java.io.IOException;
-import java.sql.Timestamp;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.concurrent.Future;
-import java.util.logging.Level;
-import java.util.logging.Logger;
+import org.apache.commons.lang3.StringUtils;
+
 import javax.ejb.EJB;
 import javax.ejb.EJBException;
 import javax.ejb.Stateless;
@@ -42,8 +24,12 @@
 import javax.persistence.PersistenceContext;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.solr.client.solrj.SolrServerException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import static edu.harvard.iq.dataverse.batch.jobs.importer.filesystem.FileRecordJobListener.SEP;
 
 /**
  *
@@ -158,18 +144,6 @@ public DatasetVersion getDatasetVersion(){
         }                
     } // end RetrieveDatasetVersionResponse
 
-    /**
-     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
-     */
-    public enum FileMetadatasOrderCriteria {
-        NameAZ,
-        NameZA,
-        Newest,
-        Oldest,
-        Size,
-        Type
-    }
-
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -1244,74 +1218,4 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
-
-    /**
-     * Returns a FileMetadata list of files in the specified DatasetVersion
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @param limit          for pagination, can be null
-     * @param offset         for pagination, can be null
-     * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
-     * @return a FileMetadata list of the specified DatasetVersion
-     */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion,
-                                               Integer limit,
-                                               Integer offset,
-                                               String fileType,
-                                               String fileAccess,
-                                               String fileTag,
-                                               FileMetadatasOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> query = createQueryFromCriteria(datasetVersion, orderCriteria);
-        if (limit != null) {
-            query.limit(limit);
-        }
-        if (offset != null) {
-            query.offset(offset);
-        }
-        return query.fetch();
-    }
-
-    // TODO: Refactor
-    private JPAQuery<FileMetadata> createQueryFromCriteria(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
-        QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
-        QDvObject dvObject = QDvObject.dvObject;
-        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<FileMetadata> query;
-        switch (orderCriteria) {
-            case NameZA:
-                query = queryFactory.selectFrom(fileMetadata)
-                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                        .orderBy(fileMetadata.label.desc());
-                break;
-            case Newest:
-                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject)
-                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                        .where(dvObject.id.eq(fileMetadata.dataFile.id))
-                        .orderBy(orderByLifetimeExpression.desc());
-                break;
-            case Oldest:
-                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject)
-                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                        .where(dvObject.id.eq(fileMetadata.dataFile.id))
-                        .orderBy(orderByLifetimeExpression.asc());
-                break;
-            case Size:
-                query = queryFactory.selectFrom(fileMetadata)
-                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                        .orderBy(fileMetadata.dataFile.filesize.asc());
-                break;
-            case Type:
-                query = queryFactory.selectFrom(fileMetadata)
-                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                        .orderBy(fileMetadata.dataFile.contentType.asc());
-                break;
-            default:
-                query = queryFactory.selectFrom(fileMetadata)
-                        .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                        .orderBy(fileMetadata.label.asc());
-                break;
-        }
-        return query;
-    }
 } // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 4de6e410b82..47e1d909986 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -241,6 +241,9 @@ public class Datasets extends AbstractApiBean {
     @Inject
     PrivateUrlServiceBean privateUrlService;
 
+    @Inject
+    DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
+
     /**
      * Used to consolidate the way we parse and handle dataset versions.
      * @param <T> 
@@ -506,13 +509,13 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @Context HttpHeaders headers) {
         return response( req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
             try {
-                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameAZ;
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileType, fileAccess, fileTag, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, fileType, fileAccess, fileTag, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 282017462e4..7a38d34b06b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -9,6 +9,7 @@
 
 import java.util.logging.Logger;
 
+import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
 import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -3333,7 +3334,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[0].label", equalTo(testFileName5));
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3344,7 +3345,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3355,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3366,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3377,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())

From 22f1cf0a19868b9db192a2241bc42de4e385ab7c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Aug 2023 09:21:39 +0100
Subject: [PATCH 283/922] Refactor: DatasetVersionFilesServiceBean

---
 .../DatasetVersionFilesServiceBean.java       | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 70340e7cb29..6154f4f88ad 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -27,6 +27,9 @@ public class DatasetVersionFilesServiceBean implements Serializable {
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
+    private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
+    private final QDvObject dvObject = QDvObject.dvObject;
+
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
      */
@@ -44,7 +47,7 @@ public enum FileMetadatasOrderCriteria {
      * @return a FileMetadata list of the specified DatasetVersion
      */
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String fileType, String fileAccess, String fileTag, FileMetadatasOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> query = createQueryFromCriteria(datasetVersion, orderCriteria);
+        JPAQuery<FileMetadata> query = createQueryFromOrderCriteria(datasetVersion, orderCriteria);
         if (limit != null) {
             query.limit(limit);
         }
@@ -54,33 +57,38 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         return query.fetch();
     }
 
-    // TODO: Refactor
-    private JPAQuery<FileMetadata> createQueryFromCriteria(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
-        QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
-        QDvObject dvObject = QDvObject.dvObject;
+    private JPAQuery<FileMetadata> createQueryFromOrderCriteria(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
         DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
         JPAQuery<FileMetadata> query;
         switch (orderCriteria) {
             case NameZA:
-                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.label.desc());
+                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.label.desc());
                 break;
             case Newest:
-                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id)).orderBy(orderByLifetimeExpression.desc());
+                query = createFileMetadataDvObjectBaseQuery(datasetVersion, queryFactory).orderBy(orderByLifetimeExpression.desc());
                 break;
             case Oldest:
-                query = queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id)).orderBy(orderByLifetimeExpression.asc());
+                query = createFileMetadataDvObjectBaseQuery(datasetVersion, queryFactory).orderBy(orderByLifetimeExpression.asc());
                 break;
             case Size:
-                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.dataFile.filesize.asc());
+                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.dataFile.filesize.asc());
                 break;
             case Type:
-                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.dataFile.contentType.asc());
+                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.dataFile.contentType.asc());
                 break;
             default:
-                query = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).orderBy(fileMetadata.label.asc());
+                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.label.asc());
                 break;
         }
         return query;
     }
-} // end class
+
+    private JPAQuery<FileMetadata> createFileMetadataDvObjectBaseQuery(DatasetVersion datasetVersion, JPAQueryFactory queryFactory) {
+        return queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id));
+    }
+
+    private JPAQuery<FileMetadata> createFileMetadataBaseQuery(DatasetVersion datasetVersion, JPAQueryFactory queryFactory) {
+        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+    }
+}

From be8b0008e78002942e0106381f8d7af517844217 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Aug 2023 10:08:29 +0100
Subject: [PATCH 284/922] Added: content type filter to getVersionFiles API
 endpoint

---
 .../DatasetVersionFilesServiceBean.java       |  6 +++-
 .../harvard/iq/dataverse/api/Datasets.java    |  4 +--
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 28 +++++++++++++------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  5 +++-
 4 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 6154f4f88ad..b1bfd5c67ce 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -43,11 +43,15 @@ public enum FileMetadatasOrderCriteria {
      * @param datasetVersion the DatasetVersion to access
      * @param limit          for pagination, can be null
      * @param offset         for pagination, can be null
+     * @param contentType    for retrieving only files with this content type
      * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
      * @return a FileMetadata list of the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String fileType, String fileAccess, String fileTag, FileMetadatasOrderCriteria orderCriteria) {
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, String fileAccess, String fileTag, FileMetadatasOrderCriteria orderCriteria) {
         JPAQuery<FileMetadata> query = createQueryFromOrderCriteria(datasetVersion, orderCriteria);
+        if (contentType != null) {
+            query.where(fileMetadata.dataFile.contentType.eq(contentType));
+        }
         if (limit != null) {
             query.limit(limit);
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 47e1d909986..4f7f41ec75f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -501,7 +501,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @PathParam("versionId") String versionId,
                                     @QueryParam("limit") Integer limit,
                                     @QueryParam("offset") Integer offset,
-                                    @QueryParam("fileType") String fileType,
+                                    @QueryParam("contentType") String contentType,
                                     @QueryParam("fileAccess") String fileAccess,
                                     @QueryParam("fileTag") String fileTag,
                                     @QueryParam("orderCriteria") String orderCriteria,
@@ -515,7 +515,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, fileType, fileAccess, fileTag, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, fileAccess, fileTag, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 7a38d34b06b..1bf6a181436 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3302,7 +3302,7 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, apiToken);
 
         int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
@@ -3313,7 +3313,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[1].label", equalTo(testFileName2));
 
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, apiToken);
 
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
@@ -3324,7 +3324,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[1].label", equalTo(testFileName4));
 
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, apiToken);
 
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
@@ -3334,7 +3334,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[0].label", equalTo(testFileName5));
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3345,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,9 +3390,19 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
+
+        // Test Content Type
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, apiToken);
+
+        fileMetadatasCount = getVersionFilesResponseContentType.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        getVersionFilesResponseContentType.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName5));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 2151a68f7da..4886624a4d8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3249,7 +3249,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3259,6 +3259,9 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (offset != null) {
             requestSpecification = requestSpecification.queryParam("offset", offset);
         }
+        if (contentType != null) {
+            requestSpecification = requestSpecification.queryParam("contentType", contentType);
+        }
         if (orderCriteria != null) {
             requestSpecification = requestSpecification.queryParam("orderCriteria", orderCriteria);
         }

From 7db09fbc2bbadd893cb60d19ad4f460842f70c6e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Aug 2023 11:37:47 +0100
Subject: [PATCH 285/922] Refactor: DatasetVersionFilesServiceBean query
 building

---
 .../DatasetVersionFilesServiceBean.java       | 48 +++++++++++--------
 .../harvard/iq/dataverse/api/Datasets.java    |  4 +-
 2 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index b1bfd5c67ce..9e51d5fb39f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse;
 
+import edu.harvard.iq.dataverse.QDataFileCategory;
 import edu.harvard.iq.dataverse.QDvObject;
 import edu.harvard.iq.dataverse.QFileMetadata;
 
@@ -29,6 +30,7 @@ public class DatasetVersionFilesServiceBean implements Serializable {
 
     private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
     private final QDvObject dvObject = QDvObject.dvObject;
+    private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
 
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
@@ -47,52 +49,58 @@ public enum FileMetadatasOrderCriteria {
      * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
      * @return a FileMetadata list of the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, String fileAccess, String fileTag, FileMetadatasOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> query = createQueryFromOrderCriteria(datasetVersion, orderCriteria);
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, String fileAccess, String categoryName, FileMetadatasOrderCriteria orderCriteria) {
+        JPAQuery<FileMetadata> query = createBaseQuery(datasetVersion, orderCriteria);
+
         if (contentType != null) {
             query.where(fileMetadata.dataFile.contentType.eq(contentType));
         }
+//        TODO
+//        if (categoryName != null) {
+//            query.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
+//        }
+
+        applyOrderCriteriaToQuery(query, orderCriteria);
+
         if (limit != null) {
             query.limit(limit);
         }
         if (offset != null) {
             query.offset(offset);
         }
+
         return query.fetch();
     }
 
-    private JPAQuery<FileMetadata> createQueryFromOrderCriteria(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
-        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
+    private JPAQuery<FileMetadata> createBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<FileMetadata> query;
+        if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) {
+            return queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id));
+        }
+        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+    }
+
+    private void applyOrderCriteriaToQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
+        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
         switch (orderCriteria) {
             case NameZA:
-                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.label.desc());
+                query.orderBy(fileMetadata.label.desc());
                 break;
             case Newest:
-                query = createFileMetadataDvObjectBaseQuery(datasetVersion, queryFactory).orderBy(orderByLifetimeExpression.desc());
+                query.orderBy(orderByLifetimeExpression.desc());
                 break;
             case Oldest:
-                query = createFileMetadataDvObjectBaseQuery(datasetVersion, queryFactory).orderBy(orderByLifetimeExpression.asc());
+                query.orderBy(orderByLifetimeExpression.asc());
                 break;
             case Size:
-                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.dataFile.filesize.asc());
+                query.orderBy(fileMetadata.dataFile.filesize.asc());
                 break;
             case Type:
-                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.dataFile.contentType.asc());
+                query.orderBy(fileMetadata.dataFile.contentType.asc());
                 break;
             default:
-                query = createFileMetadataBaseQuery(datasetVersion, queryFactory).orderBy(fileMetadata.label.asc());
+                query.orderBy(fileMetadata.label.asc());
                 break;
         }
-        return query;
-    }
-
-    private JPAQuery<FileMetadata> createFileMetadataDvObjectBaseQuery(DatasetVersion datasetVersion, JPAQueryFactory queryFactory) {
-        return queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id));
-    }
-
-    private JPAQuery<FileMetadata> createFileMetadataBaseQuery(DatasetVersion datasetVersion, JPAQueryFactory queryFactory) {
-        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 4f7f41ec75f..8548012dab2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -503,7 +503,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @QueryParam("offset") Integer offset,
                                     @QueryParam("contentType") String contentType,
                                     @QueryParam("fileAccess") String fileAccess,
-                                    @QueryParam("fileTag") String fileTag,
+                                    @QueryParam("categoryName") String categoryName,
                                     @QueryParam("orderCriteria") String orderCriteria,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
@@ -515,7 +515,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, fileAccess, fileTag, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, fileAccess, categoryName, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     

From 772a71222151f1e25ae96bfa0042996f923cab49 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 14 Aug 2023 15:56:42 +0100
Subject: [PATCH 286/922] Added: category name filter to getVersionFiles API
 endpoint

---
 .../DatasetVersionFilesServiceBean.java       | 27 ++++++++---------
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 29 ++++++++++++-------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  5 +++-
 3 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 9e51d5fb39f..fab117c8208 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -46,38 +46,39 @@ public enum FileMetadatasOrderCriteria {
      * @param limit          for pagination, can be null
      * @param offset         for pagination, can be null
      * @param contentType    for retrieving only files with this content type
+     * @param categoryName   for retrieving only files categorized with this category name
      * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
-     * @return a FileMetadata list of the specified DatasetVersion
+     * @return a FileMetadata list from the specified DatasetVersion
      */
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, String fileAccess, String categoryName, FileMetadatasOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> query = createBaseQuery(datasetVersion, orderCriteria);
+        JPAQuery<FileMetadata> baseQuery = createBaseQuery(datasetVersion, orderCriteria);
 
         if (contentType != null) {
-            query.where(fileMetadata.dataFile.contentType.eq(contentType));
+            baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
+        }
+        if (categoryName != null) {
+            baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
         }
-//        TODO
-//        if (categoryName != null) {
-//            query.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
-//        }
 
-        applyOrderCriteriaToQuery(query, orderCriteria);
+        applyOrderCriteriaToQuery(baseQuery, orderCriteria);
 
         if (limit != null) {
-            query.limit(limit);
+            baseQuery.limit(limit);
         }
         if (offset != null) {
-            query.offset(offset);
+            baseQuery.offset(offset);
         }
 
-        return query.fetch();
+        return baseQuery.fetch();
     }
 
     private JPAQuery<FileMetadata> createBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
         if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) {
-            return queryFactory.select(fileMetadata).from(fileMetadata, dvObject).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).where(dvObject.id.eq(fileMetadata.dataFile.id));
+            baseQuery.from(dvObject).where(dvObject.id.eq(fileMetadata.dataFile.id));
         }
-        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+        return baseQuery;
     }
 
     private void applyOrderCriteriaToQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 1bf6a181436..ff45f12b139 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3302,7 +3302,7 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, apiToken);
 
         int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
@@ -3313,7 +3313,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[1].label", equalTo(testFileName2));
 
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, apiToken);
 
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
@@ -3324,7 +3324,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[1].label", equalTo(testFileName4));
 
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, apiToken);
 
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
@@ -3334,7 +3334,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[0].label", equalTo(testFileName5));
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3345,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,13 +3390,13 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
 
         // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, apiToken);
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, apiToken);
 
         fileMetadatasCount = getVersionFilesResponseContentType.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
@@ -3404,5 +3404,14 @@ public void getVersionFiles() throws IOException {
         getVersionFilesResponseContentType.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName5));
+
+        // Test Category Name
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", "nonExistentCategory", null, apiToken);
+
+        fileMetadatasCount = getVersionFilesResponseCategoryName.jsonPath().getList("data").size();
+        assertEquals(0, fileMetadatasCount);
+
+        getVersionFilesResponseCategoryName.then().assertThat()
+                .statusCode(OK.getStatusCode());
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4886624a4d8..d9b1021d670 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3249,7 +3249,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String categoryName, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3262,6 +3262,9 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (contentType != null) {
             requestSpecification = requestSpecification.queryParam("contentType", contentType);
         }
+        if (categoryName != null) {
+            requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
+        }
         if (orderCriteria != null) {
             requestSpecification = requestSpecification.queryParam("orderCriteria", orderCriteria);
         }

From 1ab692d0c9f6fd8def97f82b8520190a3c76a5ec Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 15 Aug 2023 11:32:56 +0100
Subject: [PATCH 287/922] Added: access type filter to getVersionFiles API
 endpoint (pending IT tests)

---
 .../DatasetVersionFilesServiceBean.java       | 42 +++++++++++-
 .../harvard/iq/dataverse/api/Datasets.java    | 14 ++--
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 66 ++++++++++++-------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  5 +-
 4 files changed, 94 insertions(+), 33 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index fab117c8208..97864be994c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -1,16 +1,17 @@
 package edu.harvard.iq.dataverse;
 
+import com.querydsl.core.types.dsl.*;
 import edu.harvard.iq.dataverse.QDataFileCategory;
 import edu.harvard.iq.dataverse.QDvObject;
+import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
 
-import com.querydsl.core.types.dsl.CaseBuilder;
-import com.querydsl.core.types.dsl.DateTimeExpression;
 import com.querydsl.jpa.impl.JPAQuery;
 import com.querydsl.jpa.impl.JPAQueryFactory;
 
 import java.io.Serializable;
 import java.sql.Timestamp;
+import java.time.LocalDate;
 import java.util.List;
 import javax.ejb.Stateless;
 import javax.inject.Named;
@@ -39,6 +40,13 @@ public enum FileMetadatasOrderCriteria {
         NameAZ, NameZA, Newest, Oldest, Size, Type
     }
 
+    /**
+     * Status of the particular DataFile based on active embargoes and restriction state used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
+     */
+    public enum DataFileAccessStatus {
+        Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
+    }
+
     /**
      * Returns a FileMetadata list of files in the specified DatasetVersion
      *
@@ -46,16 +54,20 @@ public enum FileMetadatasOrderCriteria {
      * @param limit          for pagination, can be null
      * @param offset         for pagination, can be null
      * @param contentType    for retrieving only files with this content type
+     * @param accessStatus   for retrieving only files with this DataFileAccessStatus
      * @param categoryName   for retrieving only files categorized with this category name
      * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
      * @return a FileMetadata list from the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, String fileAccess, String categoryName, FileMetadatasOrderCriteria orderCriteria) {
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, FileMetadatasOrderCriteria orderCriteria) {
         JPAQuery<FileMetadata> baseQuery = createBaseQuery(datasetVersion, orderCriteria);
 
         if (contentType != null) {
             baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
         }
+        if (accessStatus != null) {
+            baseQuery.where(createAccessStatusExpression(accessStatus));
+        }
         if (categoryName != null) {
             baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
         }
@@ -81,6 +93,30 @@ private JPAQuery<FileMetadata> createBaseQuery(DatasetVersion datasetVersion, Fi
         return baseQuery;
     }
 
+    private BooleanExpression createAccessStatusExpression(DataFileAccessStatus accessStatus) {
+        QEmbargo embargo = fileMetadata.dataFile.embargo;
+        BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
+        BooleanExpression inactivelyEmbargoedExpression = embargo.dateAvailable.isNull().or(embargo.dateAvailable.loe(DateExpression.currentDate(LocalDate.class)));
+        BooleanExpression accessStatusExpression;
+        switch (accessStatus) {
+            case EmbargoedThenRestricted:
+                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
+                break;
+            case EmbargoedThenPublic:
+                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
+                break;
+            case Restricted:
+                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
+                break;
+            case Public:
+                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
+                break;
+            default:
+                throw new IllegalStateException("Unexpected value: " + accessStatus);
+        }
+        return accessStatusExpression;
+    }
+
     private void applyOrderCriteriaToQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
         DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
         switch (orderCriteria) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 8548012dab2..c2b6631735d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -492,7 +492,7 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
                     : ok(json(dsv));
         }, getRequestUser(crc));
     }
-    
+
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
@@ -502,12 +502,12 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @QueryParam("limit") Integer limit,
                                     @QueryParam("offset") Integer offset,
                                     @QueryParam("contentType") String contentType,
-                                    @QueryParam("fileAccess") String fileAccess,
+                                    @QueryParam("accessStatus") String accessStatus,
                                     @QueryParam("categoryName") String categoryName,
                                     @QueryParam("orderCriteria") String orderCriteria,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
-        return response( req -> {
+        return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
             try {
@@ -515,7 +515,13 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, fileAccess, categoryName, fileMetadatasOrderCriteria)));
+            DatasetVersionFilesServiceBean.DataFileAccessStatus dataFileAccessStatus;
+            try {
+                dataFileAccessStatus = accessStatus != null ? DatasetVersionFilesServiceBean.DataFileAccessStatus.valueOf(accessStatus) : null;
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
+            }
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index ff45f12b139..3c4b23a5e4d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3270,7 +3270,7 @@ public void getDatasetVersionCitation() {
     }
 
     @Test
-    public void getVersionFiles() throws IOException {
+    public void getVersionFiles() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3302,39 +3302,41 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, apiToken);
-
-        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(testPageSize, fileMetadatasCount);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName1))
                 .body("data[1].label", equalTo(testFileName2));
 
-        // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, apiToken);
-
-        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
 
+        String testFileId1 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
+
+        // Test page 2
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, apiToken);
+
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName3))
                 .body("data[1].label", equalTo(testFileName4));
 
-        // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, apiToken);
-
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
+        assertEquals(testPageSize, fileMetadatasCount);
+
+        // Test page 3 (last)
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName5));
 
+        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3347,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3358,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3369,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3380,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,28 +3392,42 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
 
         // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, apiToken);
-
-        fileMetadatasCount = getVersionFilesResponseContentType.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, apiToken);
 
         getVersionFilesResponseContentType.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName5));
 
+        fileMetadatasCount = getVersionFilesResponseContentType.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
         // Test Category Name
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", "nonExistentCategory", null, apiToken);
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, "nonExistentCategory", null, apiToken);
+
+        getVersionFilesResponseCategoryName.then().assertThat()
+                .statusCode(OK.getStatusCode());
 
         fileMetadatasCount = getVersionFilesResponseCategoryName.jsonPath().getList("data").size();
         assertEquals(0, fileMetadatasCount);
 
-        getVersionFilesResponseCategoryName.then().assertThat()
+        // Test Access Status Restricted
+        Response restrictFileResponse = UtilIT.restrictFile(String.valueOf(testFileId1), true, apiToken);
+        restrictFileResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, apiToken);
+
+        getVersionFilesResponseRestricted.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1));
+
+        fileMetadatasCount = getVersionFilesResponseRestricted.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index d9b1021d670..a36e4770c41 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3249,7 +3249,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String categoryName, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3262,6 +3262,9 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (contentType != null) {
             requestSpecification = requestSpecification.queryParam("contentType", contentType);
         }
+        if (accessStatus != null) {
+            requestSpecification = requestSpecification.queryParam("accessStatus", accessStatus);
+        }
         if (categoryName != null) {
             requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
         }

From 89da3bc91be476a9427a0022f371d2b8f31434f1 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 15 Aug 2023 16:04:41 +0200
Subject: [PATCH 288/922] chore(deps): update caffeine cache and nimbus oidc
 sdk to latest releases

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 41727f95158..270c225a2bb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -413,13 +413,13 @@
         <dependency>
             <groupId>com.nimbusds</groupId>
             <artifactId>oauth2-oidc-sdk</artifactId>
-            <version>10.9.1</version>
+            <version>10.13.2</version>
         </dependency>
         <!-- Caching library, current main use case is for OIDC authentication -->
         <dependency>
             <groupId>com.github.ben-manes.caffeine</groupId>
             <artifactId>caffeine</artifactId>
-            <version>3.1.6</version>
+            <version>3.1.8</version>
         </dependency>
         
         <!-- New and Improved GDCC XOAI library! -->

From d5d060266f0c8a6c7bf4ce6a84588182ad1d0b81 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Aug 2023 09:06:03 +0100
Subject: [PATCH 289/922] Added: embargo filters to getVersionFiles API
 endpoint

---
 .../DatasetVersionFilesServiceBean.java       |  8 ++-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 59 ++++++++++++++++++-
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 14 +++++
 3 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 97864be994c..7952cc929f4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -1,11 +1,15 @@
 package edu.harvard.iq.dataverse;
 
-import com.querydsl.core.types.dsl.*;
 import edu.harvard.iq.dataverse.QDataFileCategory;
 import edu.harvard.iq.dataverse.QDvObject;
 import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
 
+import com.querydsl.core.types.dsl.BooleanExpression;
+import com.querydsl.core.types.dsl.CaseBuilder;
+import com.querydsl.core.types.dsl.DateExpression;
+import com.querydsl.core.types.dsl.DateTimeExpression;
+
 import com.querydsl.jpa.impl.JPAQuery;
 import com.querydsl.jpa.impl.JPAQueryFactory;
 
@@ -96,7 +100,7 @@ private JPAQuery<FileMetadata> createBaseQuery(DatasetVersion datasetVersion, Fi
     private BooleanExpression createAccessStatusExpression(DataFileAccessStatus accessStatus) {
         QEmbargo embargo = fileMetadata.dataFile.embargo;
         BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
-        BooleanExpression inactivelyEmbargoedExpression = embargo.dateAvailable.isNull().or(embargo.dateAvailable.loe(DateExpression.currentDate(LocalDate.class)));
+        BooleanExpression inactivelyEmbargoedExpression = embargo.isNull();
         BooleanExpression accessStatusExpression;
         switch (accessStatus) {
             case EmbargoedThenRestricted:
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 3c4b23a5e4d..8018b2dc1c1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -7,6 +7,8 @@
 import com.jayway.restassured.http.ContentType;
 import com.jayway.restassured.response.Response;
 
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
 import java.util.logging.Logger;
 
 import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
@@ -120,6 +122,8 @@ public static void setUpClass() {
         Response removeAnonymizedFieldTypeNames = UtilIT.deleteSetting(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
         removeAnonymizedFieldTypeNames.then().assertThat()
                 .statusCode(200);
+
+        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
         
         /* With Dual mode, we can no longer mess with upload methods since native is now required for anything to work
                
@@ -147,7 +151,9 @@ public static void afterClass() {
         Response removeAnonymizedFieldTypeNames = UtilIT.deleteSetting(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
         removeAnonymizedFieldTypeNames.then().assertThat()
                 .statusCode(200);
-        
+
+        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+
         /* See above
         Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl);
         removeDcmUrl.then().assertThat()
@@ -3270,7 +3276,7 @@ public void getDatasetVersionCitation() {
     }
 
     @Test
-    public void getVersionFiles() throws IOException, InterruptedException {
+    public void getVersionFiles() throws IOException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3313,6 +3319,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
         assertEquals(testPageSize, fileMetadatasCount);
 
         String testFileId1 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
+        String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
 
         // Test page 2
         getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, apiToken);
@@ -3325,6 +3332,8 @@ public void getVersionFiles() throws IOException, InterruptedException {
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
 
+        String testFileId3 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
+
         // Test page 3 (last)
         getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, apiToken);
 
@@ -3429,5 +3438,51 @@ public void getVersionFiles() throws IOException, InterruptedException {
 
         fileMetadatasCount = getVersionFilesResponseRestricted.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
+
+        // Test Access Status Embargoed Then Public
+        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
+        String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+
+        // Create embargo for test file 1 (Embargo and Restricted)
+        Response createExpiredFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken);
+
+        createExpiredFileEmbargoResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // Create embargo for test file 2 (Embargo and Public)
+        Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken);
+
+        createActiveFileEmbargoResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, apiToken);
+
+        getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName2));
+
+        fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, apiToken);
+
+        getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1));
+
+        fileMetadatasCount = getVersionFilesResponseEmbargoedThenRestricted.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        // Test Access Status Public
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, apiToken);
+
+        getVersionFilesResponsePublic.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName3))
+                .body("data[1].label", equalTo(testFileName4))
+                .body("data[2].label", equalTo(testFileName5));
+
+        fileMetadatasCount = getVersionFilesResponsePublic.jsonPath().getList("data").size();
+        assertEquals(3, fileMetadatasCount);
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index a36e4770c41..17d227fbc56 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3304,4 +3304,18 @@ static Response getUserPermissionsOnFile(String dataFileId, String apiToken) {
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/access/datafile/" + dataFileId + "/userPermissions");
     }
+
+    static Response createFileEmbargo(Integer datasetId, Integer fileId, String dateAvailable, String apiToken) {
+        JsonObjectBuilder jsonBuilder = Json.createObjectBuilder();
+        jsonBuilder.add("dateAvailable", dateAvailable);
+        jsonBuilder.add("reason", "This is a test embargo");
+        jsonBuilder.add("fileIds", Json.createArrayBuilder().add(fileId));
+        String jsonString = jsonBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .contentType("application/json")
+                .urlEncodingEnabled(false)
+                .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
+    }
 }

From 2c3907026f888071d0240bcf060ea251a0cbe2c3 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Aug 2023 10:20:30 +0100
Subject: [PATCH 290/922] Fixed: QueryDSL to use jakarta, and missing
 after-develop-merge import added

---
 pom.xml                                        | 18 ++----------------
 .../DatasetVersionFilesServiceBean.java        |  9 +++++----
 .../edu/harvard/iq/dataverse/api/Files.java    |  2 ++
 3 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/pom.xml b/pom.xml
index f7734d093a1..321a432f1d3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -257,12 +257,14 @@
             <groupId>com.querydsl</groupId>
             <artifactId>querydsl-apt</artifactId>
             <version>${querydsl.version}</version>
+            <classifier>jakarta</classifier>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.querydsl</groupId>
             <artifactId>querydsl-jpa</artifactId>
             <version>${querydsl.version}</version>
+            <classifier>jakarta</classifier>
         </dependency>
         
         <dependency>
@@ -782,22 +784,6 @@
                     <consoleOutput>true</consoleOutput>
                 </configuration>
             </plugin>
-            <plugin>
-                <groupId>com.mysema.maven</groupId>
-                <artifactId>apt-maven-plugin</artifactId>
-                <version>1.1.3</version>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>process</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>target/generated-sources/java</outputDirectory>
-                            <processor>com.querydsl.apt.jpa.JPAAnnotationProcessor</processor>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
         </plugins>
     </build>
     <profiles>
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 7952cc929f4..9dc088dcc41 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -13,14 +13,15 @@
 import com.querydsl.jpa.impl.JPAQuery;
 import com.querydsl.jpa.impl.JPAQueryFactory;
 
+import jakarta.ejb.Stateless;
+import jakarta.inject.Named;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.PersistenceContext;
+
 import java.io.Serializable;
 import java.sql.Timestamp;
 import java.time.LocalDate;
 import java.util.List;
-import javax.ejb.Stateless;
-import javax.inject.Named;
-import javax.persistence.EntityManager;
-import javax.persistence.PersistenceContext;
 
 /**
  * @author Guillermo Portas
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index cdf30fe9389..4c411a631f1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -76,6 +76,8 @@
 import jakarta.ws.rs.core.HttpHeaders;
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
+
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonDT;
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 import jakarta.ws.rs.core.UriInfo;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;

From 50df166fcd5769b017b5dd796045266e1eade249 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Aug 2023 10:34:11 +0100
Subject: [PATCH 291/922] Added: docs and release notes

---
 .../9714-files-api-extension-filters.md       |  6 ++++++
 doc/sphinx-guides/source/api/native-api.rst   | 19 +++++++++++++++++--
 .../DatasetVersionFilesServiceBean.java       |  4 ----
 3 files changed, 23 insertions(+), 6 deletions(-)
 create mode 100644 doc/release-notes/9714-files-api-extension-filters.md

diff --git a/doc/release-notes/9714-files-api-extension-filters.md b/doc/release-notes/9714-files-api-extension-filters.md
new file mode 100644
index 00000000000..9e1e1d40999
--- /dev/null
+++ b/doc/release-notes/9714-files-api-extension-filters.md
@@ -0,0 +1,6 @@
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support filtering by:
+
+- Public
+- Restricted
+- EmbargoedThenRestricted
+- EmbargoedThenPublic
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 214ca440d1d..a175e7b5f54 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -964,6 +964,21 @@ This endpoint supports optional pagination, through the ``limit`` and ``offset``
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
 
+File filtering is optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?accessStatus=Public"
+
 Ordering criteria for sorting the results is also optionally supported. In particular, by the following possible values:
 
 * ``NameAZ`` (Default)
@@ -973,14 +988,14 @@ Ordering criteria for sorting the results is also optionally supported. In parti
 * ``Size``
 * ``Type``
 
-Please note that these values are case sensitive and must be correctly typed for the endpoint to recognize them.
-
 Usage example:
 
 .. code-block:: bash
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?orderCriteria=Newest"
 
+Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 9dc088dcc41..37f56d10b3c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -23,10 +23,6 @@
 import java.time.LocalDate;
 import java.util.List;
 
-/**
- * @author Guillermo Portas
- * TODO
- */
 @Stateless
 @Named
 public class DatasetVersionFilesServiceBean implements Serializable {

From 7153e54225daa380c7e3041535c20c2d784e902a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Aug 2023 11:10:32 +0100
Subject: [PATCH 292/922] Added: missing docs

---
 .../9714-files-api-extension-filters.md       | 18 ++++++++++++-----
 doc/sphinx-guides/source/api/native-api.rst   | 20 ++++++++++++++++++-
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/doc/release-notes/9714-files-api-extension-filters.md b/doc/release-notes/9714-files-api-extension-filters.md
index 9e1e1d40999..034230efe61 100644
--- a/doc/release-notes/9714-files-api-extension-filters.md
+++ b/doc/release-notes/9714-files-api-extension-filters.md
@@ -1,6 +1,14 @@
-The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support filtering by:
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by:
 
-- Public
-- Restricted
-- EmbargoedThenRestricted
-- EmbargoedThenPublic
+- Access status: through the `accessStatus` query parameter, which supports the following values:
+
+  - Public
+  - Restricted
+  - EmbargoedThenRestricted
+  - EmbargoedThenPublic
+
+
+- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
+
+
+- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index a175e7b5f54..3482df2816c 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -964,7 +964,23 @@ This endpoint supports optional pagination, through the ``limit`` and ``offset``
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
 
-File filtering is optionally supported. In particular, by the following possible values:
+Category name filtering is also optionally supported. To return files to which the requested category has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?categoryName=Data"
+
+Content type filtering is also optionally supported. To return files matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?contentType=image/png"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
 
 * ``Public``
 * ``Restricted``
@@ -996,6 +1012,8 @@ Usage example:
 
 Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
 
+Keep in mind that you can combine all of the above query params depending on the results you are looking for.
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 35e454767288d191d496b04a2ee0f6471bebac53 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 16 Aug 2023 20:16:59 +0100
Subject: [PATCH 293/922] Fixed: missing import removed by mistake after
 develop merge

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index cdf30fe9389..4c411a631f1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -76,6 +76,8 @@
 import jakarta.ws.rs.core.HttpHeaders;
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
+
+import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonDT;
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 import jakarta.ws.rs.core.UriInfo;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;

From 4b755b50bfbe729570dde943c1809ef80b3b840f Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 18 Aug 2023 17:25:52 -0400
Subject: [PATCH 294/922] setting is GlobusAppUrl not ...URL

---
 doc/sphinx-guides/source/installation/config.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 8493702406b..a5579c82c6d 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -3926,7 +3926,7 @@ GlobusEndpoint is Globus endpoint id used with Globus integration. See :ref:`glo
 
 A comma-separated list of the S3 stores that are configured to support Globus integration. See :ref:`globus-support` for details.
 
-:GlobusAppURL
+:GlobusAppUrl
 +++++++++++++
 
 The URL where the `dataverse-globus <https://github.com/scholarsportal/dataverse-globus>`_ "transfer" app has been deployed to support Globus integration. See :ref:`globus-support` for details.

From a835f5db1dc2ed3fd307c012d8b1535dae24523f Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 18 Aug 2023 18:42:30 -0400
Subject: [PATCH 295/922] added pagination to the /versions api. dropped the
 files section from the (default) output of the api. (#9763)

---
 doc/sphinx-guides/source/api/native-api.rst   |   9 +-
 .../iq/dataverse/DatasetServiceBean.java      |   2 +-
 .../harvard/iq/dataverse/DatasetVersion.java  |   6 +-
 .../dataverse/DatasetVersionServiceBean.java  | 112 +++++++++++++++++-
 .../harvard/iq/dataverse/api/Datasets.java    |  36 +++---
 .../iq/dataverse/dataset/DatasetUtil.java     |   2 +-
 .../command/impl/ListVersionsCommand.java     |  48 +++++---
 .../iq/dataverse/util/json/JsonPrinter.java   |  16 +--
 8 files changed, 188 insertions(+), 43 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 4d9466703e4..da3fbfffa73 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -830,7 +830,7 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/datasets/24/versions"
+  curl "https://demo.dataverse.org/api/datasets/24/versions?includeFiles=true"
 
 It returns a list of versions with their metadata, and file list:
 
@@ -883,6 +883,10 @@ It returns a list of versions with their metadata, and file list:
     ]
   }
 
+The optional ``includeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``false``. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. 
+
+The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. 
+
 
 Get Version of a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~
@@ -903,6 +907,9 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0"
 
+The optional ``includeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``false``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. 
+
+
 .. _export-dataset-metadata-api:
 
 Export Metadata of a Dataset in Various Formats
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
index 52eb5868c35..ceb5902defa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
@@ -137,7 +137,7 @@ public Dataset findDeep(Object pk) {
             .setHint("eclipselink.left-join-fetch", "o.files.roleAssignments")
             .getSingleResult();
     }
-
+    
     public List<Dataset> findByOwnerId(Long ownerId) {
         return findByOwnerId(ownerId, false);
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
index 5836bd9e175..8d4dafad62a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
@@ -68,7 +68,11 @@
                query = "SELECT OBJECT(o) FROM DatasetVersion AS o WHERE o.dataset.harvestedFrom IS NULL and o.releaseTime IS NOT NULL and o.archivalCopyLocation IS NULL"
     ), 
     @NamedQuery(name = "DatasetVersion.findById", 
-                query = "SELECT o FROM DatasetVersion o LEFT JOIN FETCH o.fileMetadatas WHERE o.id=:id")})
+                query = "SELECT o FROM DatasetVersion o LEFT JOIN FETCH o.fileMetadatas WHERE o.id=:id"), 
+    @NamedQuery(name = "DatasetVersion.findByDataset",
+                query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC"), 
+    @NamedQuery(name = "DatasetVersion.findReleasedByDataset",
+                query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId AND o.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC")})
     
     
 @Entity
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 28243c37eee..27a4f4773d4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -48,7 +48,23 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
     private static final Logger logger = Logger.getLogger(DatasetVersionServiceBean.class.getCanonicalName());
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
-    
+
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
+            + " WHERE fm.datasetVersion.id=:datasetVersionId"
+            + " ORDER BY fm.label";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = dvo.id"
+            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.filesize";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.contentType";
+
     @EJB
     DatasetServiceBean datasetService;
     
@@ -149,7 +165,19 @@ public DatasetVersion getDatasetVersion(){
             return this.datasetVersionForResponse;
         }                
     } // end RetrieveDatasetVersionResponse
-    
+
+    /**
+     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
+     */
+    public enum FileMetadatasOrderCriteria {
+        NameAZ,
+        NameZA,
+        Newest,
+        Oldest,
+        Size,
+        Type
+    }
+
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -168,7 +196,39 @@ public DatasetVersion findDeep(Object pk) {
             .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.creator")
             .getSingleResult();
     }
-
+    
+    /**
+     * Performs the same database lookup as the one behind Dataset.getVersions().
+     * Additionally, provides the arguments for selecting a partial list of 
+     * (length-offset) versions for pagination, plus the ability to pre-select 
+     * only the publicly-viewable versions. 
+     * @param datasetId
+     * @param offset for pagination through long lists of versions
+     * @param length for pagination through long lists of versions
+     * @param includeUnpublished retrieves all the versions, including drafts and deaccessioned. 
+     * @return (partial) list of versions
+     */
+    public List<DatasetVersion> findVersions(Long datasetId, Integer offset, Integer length, boolean includeUnpublished) {
+        TypedQuery<DatasetVersion> query;  
+        if (includeUnpublished) {
+            query = em.createNamedQuery("DatasetVersion.findByDataset", DatasetVersion.class);
+        } else {
+            query = em.createNamedQuery("DatasetVersion.findReleasedByDataset", DatasetVersion.class)
+                    .setParameter("datasetId", datasetId);
+        }
+        
+        query.setParameter("datasetId", datasetId);
+        
+        if (offset != null) {
+            query.setFirstResult(offset);
+        }
+        if (length != null) {
+            query.setMaxResults(length);
+        }
+        
+        return query.getResultList();
+    }
+    
     public DatasetVersion findByFriendlyVersionNumber(Long datasetId, String friendlyVersionNumber) {
         Long majorVersionNumber = null;
         Long minorVersionNumber = null;
@@ -1224,4 +1284,50 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
+
+    /**
+     * Returns a FileMetadata list of files in the specified DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param limit for pagination, can be null
+     * @param offset for pagination, can be null
+     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
+     * @return a FileMetadata list of the specified DatasetVersion
+     */
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
+        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
+                .setParameter("datasetVersionId", datasetVersion.getId());
+        if (limit != null) {
+            query.setMaxResults(limit);
+        }
+        if (offset != null) {
+            query.setFirstResult(offset);
+        }
+        return query.getResultList();
+    }
+
+    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
+        String queryString;
+        switch (orderCriteria) {
+            case NameZA:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
+                break;
+            case Newest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
+                break;
+            case Oldest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
+                break;
+            case Size:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
+                break;
+            case Type:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
+                break;
+            default:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
+                break;
+        }
+        return queryString;
+    }
 } // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index dbea63cb1c8..25d077f9807 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -260,7 +260,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id"
                 MakeDataCountLoggingServiceBean.MakeDataCountEntry entry = new MakeDataCountEntry(uriInfo, headers, dvRequestService, retrieved);
                 mdcLogService.logEntry(entry);
             }
-            return ok(jsonbuilder.add("latestVersion", (latest != null) ? json(latest) : null));
+            return ok(jsonbuilder.add("latestVersion", (latest != null) ? json(latest, true) : null));
         }, getRequestUser(crc));
     }
     
@@ -466,31 +466,39 @@ public Response useDefaultCitationDate(@Context ContainerRequestContext crc, @Pa
     @GET
     @AuthRequired
     @Path("{id}/versions")
-    public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id ) {
+    public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("includeFiles") Boolean includeFiles, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) {
         return response( req ->
-             ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id)) )
+             ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id), offset, limit) )
                                 .stream()
-                                .map( d -> json(d) )
+                                .map( d -> json(d, includeFiles == null ? false : includeFiles) )
                                 .collect(toJsonArray())), getRequestUser(crc));
     }
     
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}")
-    public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+    public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("includeFiles") Boolean includeFiles, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         return response( req -> {
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
-                    : ok(json(dsv));
+                    : ok(json(dsv, includeFiles == null ? false : includeFiles));
         }, getRequestUser(crc));
     }
     
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
-    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return response( req -> ok( jsonFileMetadatas(
-                         getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getFileMetadatas())), getRequestUser(crc));
+    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response( req -> {
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            try {
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
+            }
+            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileMetadatasOrderCriteria)));
+        }, getRequestUser(crc));
     }
     
     @GET
@@ -708,7 +716,7 @@ public Response updateDraftVersion(@Context ContainerRequestContext crc, String
                 }
                 managedVersion = execCommand(new CreateDatasetVersionCommand(req, ds, incomingVersion));
             }
-            return ok( json(managedVersion) );
+            return ok( json(managedVersion, true) );
                     
         } catch (JsonParseException ex) {
             logger.log(Level.SEVERE, "Semantic error parsing dataset version Json: " + ex.getMessage(), ex);
@@ -943,7 +951,7 @@ private Response processDatasetFieldDataDelete(String jsonBody, String id, Datav
 
 
             DatasetVersion managedVersion = execCommand(new UpdateDatasetVersionCommand(ds, req)).getLatestVersion();
-            return ok(json(managedVersion));
+            return ok(json(managedVersion, true));
 
         } catch (JsonParseException ex) {
             logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + ex.getMessage(), ex);
@@ -1092,7 +1100,7 @@ private Response processDatasetUpdate(String jsonBody, String id, DataverseReque
             }
             DatasetVersion managedVersion = execCommand(new UpdateDatasetVersionCommand(ds, req)).getLatestVersion();
 
-            return ok(json(managedVersion));
+            return ok(json(managedVersion, true));
 
         } catch (JsonParseException ex) {
             logger.log(Level.SEVERE, "Semantic error parsing dataset update Json: " + ex.getMessage(), ex);
@@ -3848,9 +3856,9 @@ public Response getPrivateUrlDatasetVersion(@PathParam("privateUrlToken") String
         JsonObjectBuilder responseJson;
         if (isAnonymizedAccess) {
             List<String> anonymizedFieldTypeNamesList = new ArrayList<>(Arrays.asList(anonymizedFieldTypeNames.split(",\\s")));
-            responseJson = json(dsv, anonymizedFieldTypeNamesList);
+            responseJson = json(dsv, anonymizedFieldTypeNamesList, true);
         } else {
-            responseJson = json(dsv);
+            responseJson = json(dsv, true);
         }
         return ok(responseJson);
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
index adbd132bce8..e36ba34a364 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
@@ -521,7 +521,7 @@ public static boolean validateDatasetMetadataExternally(Dataset ds, String execu
         // for the filter to whitelist by these attributes. 
         
         try {
-            jsonMetadata = json(ds).add("datasetVersion", json(ds.getLatestVersion()))
+            jsonMetadata = json(ds).add("datasetVersion", json(ds.getLatestVersion(), true))
                     .add("sourceAddress", sourceAddressLabel)
                     .add("userIdentifier", userIdentifier)
                     .add("parentAlias", ds.getOwner().getAlias())
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
index 51283f29156..80a5fe9b080 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
@@ -23,23 +23,41 @@
  */
 // No permission needed to view published dvObjects
 @RequiredPermissions({})
-public class ListVersionsCommand extends AbstractCommand<List<DatasetVersion>>{
-    
+public class ListVersionsCommand extends AbstractCommand<List<DatasetVersion>> {
+
     private final Dataset ds;
+    private final Integer limit; 
+    private final Integer offset;
     
-	public ListVersionsCommand(DataverseRequest aRequest, Dataset aDataset) {
-		super(aRequest, aDataset);
-		ds = aDataset;
-	}
+    public ListVersionsCommand(DataverseRequest aRequest, Dataset aDataset) {
+        this(aRequest, aDataset, null, null);
+    }
+
+    public ListVersionsCommand(DataverseRequest aRequest, Dataset aDataset, Integer offset, Integer limit) {
+        super(aRequest, aDataset);
+        ds = aDataset;
+        this.offset = offset; 
+        this.limit = limit; 
+    }
 
-	@Override
-	public List<DatasetVersion> execute(CommandContext ctxt) throws CommandException {
-		List<DatasetVersion> outputList = new LinkedList<>();
-		for ( DatasetVersion dsv : ds.getVersions() ) {
-            if (dsv.isReleased() || ctxt.permissions().request( getRequest() ).on(ds).has(Permission.EditDataset)) {
-                outputList.add(dsv);
+    @Override
+    public List<DatasetVersion> execute(CommandContext ctxt) throws CommandException {
+        
+        boolean includeUnpublished = ctxt.permissions().request(getRequest()).on(ds).has(Permission.EditDataset);
+        
+        if (offset == null && limit == null) {
+            // @todo: this fragment can be dropped, and the service-based method below 
+            // can be used for both cases. 
+            List<DatasetVersion> outputList = new LinkedList<>();
+            for (DatasetVersion dsv : ds.getVersions()) {
+                if (dsv.isReleased() || includeUnpublished) {
+                    outputList.add(dsv);
+                }
             }
-		}
-        return outputList;
-	}
+            return outputList;
+        } else {
+            // Only a partial list (one "page"-worth) of versions is being requested
+            return ctxt.datasetVersion().findVersions(ds.getId(), offset, limit, includeUnpublished);
+        }
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index b6026998bb7..dc8971c9539 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -368,11 +368,11 @@ public static JsonObjectBuilder json(FileDetailsHolder ds) {
                 .add("mime",ds.getMime()));
     }
 
-    public static JsonObjectBuilder json(DatasetVersion dsv) {
-        return json(dsv, null);
+    public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) {
+        return json(dsv, null, includeFiles);
     }
 
-    public static JsonObjectBuilder json(DatasetVersion dsv, List<String> anonymizedFieldTypeNamesList) {
+    public static JsonObjectBuilder json(DatasetVersion dsv, List<String> anonymizedFieldTypeNamesList, boolean includeFiles) {
         Dataset dataset = dsv.getDataset();
         JsonObjectBuilder bld = jsonObjectBuilder()
                 .add("id", dsv.getId()).add("datasetId", dataset.getId())
@@ -415,7 +415,9 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List<String> anonymized
                 jsonByBlocks(dsv.getDatasetFields(), anonymizedFieldTypeNamesList)
                 : jsonByBlocks(dsv.getDatasetFields())
         );
-        bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas()));
+        if (includeFiles) {
+            bld.add("files", jsonFileMetadatas(dsv.getFileMetadatas()));
+        }
 
         return bld;
     }
@@ -447,8 +449,8 @@ public static JsonObjectBuilder jsonDataFileList(List<DataFile> dataFiles){
      * to the regular `json` method for DatasetVersion? Will anything break?
      * Unit tests for that method could not be found.
      */
-    public static JsonObjectBuilder jsonWithCitation(DatasetVersion dsv) {
-        JsonObjectBuilder dsvWithCitation = JsonPrinter.json(dsv);
+    public static JsonObjectBuilder jsonWithCitation(DatasetVersion dsv, boolean includeFiles) {
+        JsonObjectBuilder dsvWithCitation = JsonPrinter.json(dsv, includeFiles);
         dsvWithCitation.add("citation", dsv.getCitation());
         return dsvWithCitation;
     }
@@ -467,7 +469,7 @@ public static JsonObjectBuilder jsonWithCitation(DatasetVersion dsv) {
      */
     public static JsonObjectBuilder jsonAsDatasetDto(DatasetVersion dsv) {
         JsonObjectBuilder datasetDtoAsJson = JsonPrinter.json(dsv.getDataset());
-        datasetDtoAsJson.add("datasetVersion", jsonWithCitation(dsv));
+        datasetDtoAsJson.add("datasetVersion", jsonWithCitation(dsv, true));
         return datasetDtoAsJson;
     }
 

From de35ae7c65fc4b77704ab0cd5df4a9d31ec0dbad Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 21 Aug 2023 00:17:17 -0400
Subject: [PATCH 296/922] added left join hints to the full filemetadatas
 lookup. #9763

---
 .../dataverse/DatasetVersionServiceBean.java  | 25 +++++++++++++++----
 .../harvard/iq/dataverse/api/Datasets.java    |  1 +
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 27a4f4773d4..1edc281fa3e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -202,6 +202,8 @@ public DatasetVersion findDeep(Object pk) {
      * Additionally, provides the arguments for selecting a partial list of 
      * (length-offset) versions for pagination, plus the ability to pre-select 
      * only the publicly-viewable versions. 
+     * It is recommended that individual software components utilize the 
+     * ListVersionsCommand, instead of calling this service method directly.
      * @param datasetId
      * @param offset for pagination through long lists of versions
      * @param length for pagination through long lists of versions
@@ -1297,11 +1299,24 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
         TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
                 .setParameter("datasetVersionId", datasetVersion.getId());
-        if (limit != null) {
-            query.setMaxResults(limit);
-        }
-        if (offset != null) {
-            query.setFirstResult(offset);
+        
+        if (limit == null && offset == null) {
+            query.setHint("eclipselink.left-join-fetch", "fm.dataFile.ingestRequest")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.thumbnailForDataset")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.dataTables")
+                    .setHint("eclipselink.left-join-fetch", "fm.fileCategories")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.embargo")
+                    .setHint("eclipselink.left-join-fetch", "fm.datasetVersion")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.releaseUser")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.creator");
+        } else {
+            // @todo: is there really no way to use offset-limit with left join hints?
+            if (limit != null) {
+                query.setMaxResults(limit);
+            }
+            if (offset != null) {
+                query.setFirstResult(offset);
+            }
         }
         return query.getResultList();
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 25d077f9807..48755d4ea8a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -479,6 +479,7 @@ public Response listVersions(@Context ContainerRequestContext crc, @PathParam("i
     @Path("{id}/versions/{versionId}")
     public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("includeFiles") Boolean includeFiles, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         return response( req -> {
+            // @todo: consider using DatasetVersionServiceBean.findDeep() here 
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
                     : ok(json(dsv, includeFiles == null ? false : includeFiles));

From 59de16978e97b993a72dc2d7a58ab632d1c5fa27 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 21 Aug 2023 15:28:39 +0100
Subject: [PATCH 297/922] Added: getVersionFiles searchText filtering through
 DB querying

---
 .../DatasetVersionFilesServiceBean.java       |  7 +++-
 .../harvard/iq/dataverse/api/Datasets.java    |  3 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 40 ++++++++++++-------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  5 ++-
 4 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 37f56d10b3c..751da6daba8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -57,10 +57,11 @@ public enum DataFileAccessStatus {
      * @param contentType    for retrieving only files with this content type
      * @param accessStatus   for retrieving only files with this DataFileAccessStatus
      * @param categoryName   for retrieving only files categorized with this category name
+     * @param searchText     for retrieving only files that contain the specified text within their labels or descriptions
      * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
      * @return a FileMetadata list from the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, FileMetadatasOrderCriteria orderCriteria) {
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
         JPAQuery<FileMetadata> baseQuery = createBaseQuery(datasetVersion, orderCriteria);
 
         if (contentType != null) {
@@ -72,6 +73,10 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         if (categoryName != null) {
             baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
         }
+        if (searchText != null && !searchText.isEmpty()) {
+            searchText = searchText.trim().toLowerCase();
+            baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
+        }
 
         applyOrderCriteriaToQuery(baseQuery, orderCriteria);
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a9349f2ed25..dc192281695 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -499,6 +499,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @QueryParam("contentType") String contentType,
                                     @QueryParam("accessStatus") String accessStatus,
                                     @QueryParam("categoryName") String categoryName,
+                                    @QueryParam("searchText") String searchText,
                                     @QueryParam("orderCriteria") String orderCriteria,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
@@ -516,7 +517,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
             }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, searchText, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
     
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index e24ee60d7dc..b3270f997a8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3308,7 +3308,7 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3322,7 +3322,7 @@ public void getVersionFiles() throws IOException {
         String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
 
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3335,7 +3335,7 @@ public void getVersionFiles() throws IOException {
         String testFileId3 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
 
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3345,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3389,7 +3389,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3401,13 +3401,13 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
 
         // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, apiToken);
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, apiToken);
 
         getVersionFilesResponseContentType.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3417,7 +3417,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Category Name
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, "nonExistentCategory", null, apiToken);
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, "nonExistentCategory", null, null, apiToken);
 
         getVersionFilesResponseCategoryName.then().assertThat()
                 .statusCode(OK.getStatusCode());
@@ -3430,7 +3430,7 @@ public void getVersionFiles() throws IOException {
         restrictFileResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, apiToken);
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, apiToken);
 
         getVersionFilesResponseRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3455,7 +3455,7 @@ public void getVersionFiles() throws IOException {
         createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, apiToken);
 
         getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3464,7 +3464,7 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
 
-        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, apiToken);
 
         getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3474,7 +3474,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Access Status Public
-        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, apiToken);
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, apiToken);
 
         getVersionFilesResponsePublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3484,5 +3484,15 @@ public void getVersionFiles() throws IOException {
 
         fileMetadatasCount = getVersionFilesResponsePublic.jsonPath().getList("data").size();
         assertEquals(3, fileMetadatasCount);
+
+        // Test Search Text
+        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, "test_1", null, apiToken);
+
+        getVersionFilesResponseSearchText.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1));
+
+        fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index d1ac30cfceb..8e0fb73fba8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3274,7 +3274,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String searchText, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3293,6 +3293,9 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (categoryName != null) {
             requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
         }
+        if (searchText != null) {
+            requestSpecification = requestSpecification.queryParam("searchText", searchText);
+        }
         if (orderCriteria != null) {
             requestSpecification = requestSpecification.queryParam("orderCriteria", orderCriteria);
         }

From 58964db4102be36b5455a284cfa235d799196964 Mon Sep 17 00:00:00 2001
From: Victoria Lubitch <victoria.lubitch@utoronto.ca>
Date: Mon, 21 Aug 2023 12:48:45 -0400
Subject: [PATCH 298/922] Alternative titles

---
 .../export/ddi/dataset-create-new-all-ddi-fields.json          | 2 +-
 .../java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml    | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
index 1cc2d9f761f..bdff949bb36 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
+++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json
@@ -53,7 +53,7 @@
             "typeName": "alternativeTitle",
             "multiple": true,
             "typeClass": "primitive",
-            "value": ["Alternative Title"]
+            "value": ["Alternative Title1", "Alternative Title2"]
           },
           {
             "typeName": "otherId",
diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml
index 6e2ccfd5545..507d752192d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml
+++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml
@@ -21,7 +21,8 @@
       <titlStmt>
         <titl>Replication Data for: Title</titl>
         <subTitl>Subtitle</subTitl>
-        <altTitl>Alternative Title</altTitl>
+        <altTitl>Alternative Title1</altTitl>
+        <altTitl>Alternative Title2</altTitl>
         <IDNo agency="DOI">doi:10.5072/FK2/WKUKGV</IDNo>
         <IDNo agency="OtherIDAgency1">OtherIDIdentifier1</IDNo>
         <IDNo agency="OtherIDAgency2">OtherIDIdentifier2</IDNo>

From d777f66c7f788ad57a04b3d6c5b73740770825a9 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 09:12:13 +0200
Subject: [PATCH 299/922] ci(test): update Maven ITs workflow to Java 17

---
 .github/workflows/maven_unit_test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 876d17c729e..79669c53c5d 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -79,7 +79,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
                 experimental: [ false ]
                 status: [ "Stable" ]
                 #
@@ -130,7 +130,7 @@ jobs:
           - uses: actions/checkout@v3
           - uses: actions/setup-java@v3
             with:
-                java-version: '11'
+                java-version: '17'
                 distribution: temurin
                 cache: maven
 

From 6c832a0ebcd9a72a823d3664f3136e6cae476bce Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 09:13:07 +0200
Subject: [PATCH 300/922] ci(test): trigger image builds after successfull
 integration tests, not unit tests

---
 .github/workflows/maven_unit_test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
index 79669c53c5d..a70c55fc31d 100644
--- a/.github/workflows/maven_unit_test.yml
+++ b/.github/workflows/maven_unit_test.yml
@@ -160,6 +160,6 @@ jobs:
             contents: read
             packages: write
             pull-requests: write
-        needs: unittest
+        needs: integration-test
         uses: ./.github/workflows/container_app_push.yml
         secrets: inherit

From 9ab3a75de6682aa0c8c673a7c5c0cde40298daf9 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 09:22:03 +0200
Subject: [PATCH 301/922] chore(deps): update to Testcontainers 1.19.0

---
 modules/dataverse-parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 8d0b60aaa36..61ab6626ff9 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -168,7 +168,7 @@
         <gdcc.xoai.version>5.0.0</gdcc.xoai.version>
     
         <!-- Testing dependencies -->
-        <testcontainers.version>1.18.1</testcontainers.version>
+        <testcontainers.version>1.19.0</testcontainers.version>
         <smallrye-mpconfig.version>2.10.1</smallrye-mpconfig.version>
     
         <junit.version>4.13.1</junit.version>

From 1db004245edd2f62f8d76290f4a183f095af36cc Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 14:24:55 +0200
Subject: [PATCH 302/922] build(settings): migrate ConfigCheckService to
 Jakarta EE 10

---
 .../harvard/iq/dataverse/settings/ConfigCheckService.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
index 9e2a82d6b58..b175eafc3e0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
@@ -2,10 +2,10 @@
 
 import edu.harvard.iq.dataverse.util.FileUtil;
 
-import javax.annotation.PostConstruct;
-import javax.ejb.DependsOn;
-import javax.ejb.Singleton;
-import javax.ejb.Startup;
+import jakarta.annotation.PostConstruct;
+import jakarta.ejb.DependsOn;
+import jakarta.ejb.Singleton;
+import jakarta.ejb.Startup;
 import java.io.IOException;
 import java.nio.file.FileSystemException;
 import java.nio.file.Files;

From 72722e77afb848131dc38042d23b7cf44156a6e8 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 15:01:05 +0200
Subject: [PATCH 303/922] refactor(collection): make logo location non-static
 #9662

By introducing a new static method ThemeWidgetFragment.getLogoDir
all other places (api.Access, api.Dataverse, UpdateDataverseThemeCommand,
DataverseServiceBean) can use a lookup function from one central place
instead of building the path on their own.

Reducing code duplication also means we can more easily get the location
from a setting, enabling relocation of the data. That is especially
important for container usage.

Also, we can now use ConfigCheckService to detect if the folders we configured
are read/write accessible to us.
---
 .../iq/dataverse/DataverseServiceBean.java     | 11 +----------
 .../iq/dataverse/ThemeWidgetFragment.java      | 18 ++++++++++++++----
 .../edu/harvard/iq/dataverse/api/Access.java   | 12 ++----------
 .../harvard/iq/dataverse/api/Dataverses.java   |  2 ++
 .../impl/UpdateDataverseThemeCommand.java      |  4 ++--
 5 files changed, 21 insertions(+), 26 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 7194a1ef31e..549b8310122 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -399,16 +399,7 @@ private File getLogo(Dataverse dataverse) {
         
         DataverseTheme theme = dataverse.getDataverseTheme(); 
         if (theme != null && theme.getLogo() != null && !theme.getLogo().isEmpty()) {
-            Properties p = System.getProperties();
-            String domainRoot = p.getProperty("com.sun.aas.instanceRoot");
-  
-            if (domainRoot != null && !"".equals(domainRoot)) {
-                return new File (domainRoot + File.separator + 
-                    "docroot" + File.separator + 
-                    "logos" + File.separator + 
-                    dataverse.getLogoOwnerId() + File.separator + 
-                    theme.getLogo());
-            }
+            return ThemeWidgetFragment.getLogoDir(dataverse.getLogoOwnerId()).resolve(theme.getLogo()).toFile();
         }
             
         return null;         
diff --git a/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java b/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java
index 9a62a99722a..f30051e26ae 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThemeWidgetFragment.java
@@ -7,6 +7,7 @@
 
 import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseThemeCommand;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import java.io.File;
@@ -14,6 +15,7 @@
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.logging.Level;
@@ -49,6 +51,8 @@ public class ThemeWidgetFragment implements java.io.Serializable {
     static final String DEFAULT_TEXT_COLOR = "888888";
     private static final Logger logger = Logger.getLogger(ThemeWidgetFragment.class.getCanonicalName());   
 
+    public static final String LOGOS_SUBDIR = "logos";
+    public static final String LOGOS_TEMP_SUBDIR = LOGOS_SUBDIR + File.separator + "temp";
 
     private File tempDir;
     private File uploadedFile;
@@ -86,12 +90,18 @@ public void setTaglineInput(HtmlInputText taglineInput) {
     }
 
  
-   
+    public static Path getLogoDir(String ownerId) {
+        return Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), LOGOS_SUBDIR, ownerId);
+    }
     
-    private  void createTempDir() {
+    private void createTempDir() {
           try {
-            File tempRoot = Files.createDirectories(Paths.get("../docroot/logos/temp")).toFile();
-            tempDir = Files.createTempDirectory(tempRoot.toPath(),editDv.getId().toString()).toFile();
+            // Create the temporary space if not yet existing (will silently ignore preexisting)
+            // Note that the docroot directory is checked within ConfigCheckService for presence and write access.
+            Path tempRoot = Path.of(JvmSettings.DOCROOT_DIRECTORY.lookup(), LOGOS_TEMP_SUBDIR);
+            Files.createDirectories(tempRoot);
+            
+            this.tempDir = Files.createTempDirectory(tempRoot, editDv.getId().toString()).toFile();
         } catch (IOException e) {
             throw new RuntimeException("Error creating temp directory", e); // improve error handling
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 0341f8c1127..ce7cfb6b254 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -31,6 +31,7 @@
 import edu.harvard.iq.dataverse.RoleAssignment;
 import edu.harvard.iq.dataverse.UserNotification;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
+import edu.harvard.iq.dataverse.ThemeWidgetFragment;
 
 import static edu.harvard.iq.dataverse.api.Datasets.handleVersion;
 
@@ -1196,16 +1197,7 @@ private File getLogo(Dataverse dataverse) {
         
         DataverseTheme theme = dataverse.getDataverseTheme(); 
         if (theme != null && theme.getLogo() != null && !theme.getLogo().equals("")) {
-            Properties p = System.getProperties();
-            String domainRoot = p.getProperty("com.sun.aas.instanceRoot");
-  
-            if (domainRoot != null && !"".equals(domainRoot)) {
-                return new File (domainRoot + File.separator + 
-                    "docroot" + File.separator + 
-                    "logos" + File.separator + 
-                    dataverse.getLogoOwnerId() + File.separator + 
-                    theme.getLogo());
-            }
+            return ThemeWidgetFragment.getLogoDir(dataverse.getLogoOwnerId()).resolve(theme.getLogo()).toFile();
         }
             
         return null;         
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index a60775cbd38..30c14535251 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -976,6 +976,8 @@ public Response listAssignments(@Context ContainerRequestContext crc, @PathParam
      */
 //    File tempDir;
 //
+//    TODO: Code duplicate in ThemeWidgetFragment. Maybe extract, make static and put some place else?
+//          Important: at least use JvmSettings.DOCROOT_DIRECTORY and not the hardcoded location!
 //    private void createTempDir(Dataverse editDv) {
 //        try {
 //            File tempRoot = java.nio.file.Files.createDirectories(Paths.get("../docroot/logos/temp")).toFile();
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseThemeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseThemeCommand.java
index add7b825659..9ef9fed4b1b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseThemeCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseThemeCommand.java
@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.ThemeWidgetFragment;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -22,7 +23,6 @@
 public class UpdateDataverseThemeCommand extends AbstractCommand<Dataverse> {
     private final Dataverse editedDv;
     private final File uploadedFile;
-    private final Path logoPath = Paths.get("../docroot/logos");
     private String locate;
 
     public UpdateDataverseThemeCommand(Dataverse editedDv, File uploadedFile, DataverseRequest aRequest, String location) {
@@ -44,7 +44,7 @@ public UpdateDataverseThemeCommand(Dataverse editedDv, File uploadedFile, Datave
     public Dataverse execute(CommandContext ctxt) throws CommandException {
         // Get current dataverse, so we can delete current logo file if necessary
         Dataverse currentDv = ctxt.dataverses().find(editedDv.getId());
-        File logoFileDir = new File(logoPath.toFile(), editedDv.getId().toString());
+        File logoFileDir = ThemeWidgetFragment.getLogoDir(editedDv.getId().toString()).toFile();
         File currentFile=null;
 
         if (locate.equals("FOOTER")){

From 431afed3d78fd514151e8e25c9389fcbfea79f22 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 16:12:22 +0200
Subject: [PATCH 304/922] fix(webapp): revert to hardcoded default for dirs in
 glassfish-web.xml #9662

Payara does not support looking up variables in default values of a lookup.
As a consequence, we must return to the hardcoded "./docroot" and "./uploads"
and instead supply default values using two environment variables in the
applications' Dockerfile. This way they stay configurable from cmdline or
other sources of env vars.
---
 src/main/docker/Dockerfile                         |  5 +++++
 .../META-INF/microprofile-config.properties        |  4 ++--
 src/main/webapp/WEB-INF/glassfish-web.xml          | 14 +++++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
index 88020a118b5..201f164d961 100644
--- a/src/main/docker/Dockerfile
+++ b/src/main/docker/Dockerfile
@@ -29,6 +29,11 @@ FROM $BASE_IMAGE
 # See also https://download.eclipse.org/microprofile/microprofile-config-3.0/microprofile-config-spec-3.0.html#configprofile
 ENV MP_CONFIG_PROFILE=ct
 
+# Workaround to configure upload directories by default to useful place until we can have variable lookups in
+# defaults for glassfish-web.xml and other places.
+ENV DATAVERSE_FILES_UPLOADS="${STORAGE_DIR}/uploads"
+ENV DATAVERSE_FILES_DOCROOT="${STORAGE_DIR}/docroot"
+
 # Copy app and deps from assembly in proper layers
 COPY --chown=payara:payara maven/deps ${DEPLOY_DIR}/dataverse/WEB-INF/lib/
 COPY --chown=payara:payara maven/app ${DEPLOY_DIR}/dataverse/
diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index b58c728316b..11471663fc3 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -12,8 +12,8 @@ dataverse.build=
 dataverse.files.directory=/tmp/dataverse
 # The variables are replaced with the environment variables from our base image, but still easy to override
 %ct.dataverse.files.directory=${STORAGE_DIR}
-# NOTE: the following uses STORAGE_DIR for both containers and classic installations. By defaulting to
-#       "com.sun.aas.instanceRoot" if not present, it equals the hardcoded former default "." in glassfish-web.xml
+# NOTE: The following uses STORAGE_DIR for both containers and classic installations. By defaulting to
+#       "com.sun.aas.instanceRoot" if not present, it equals the hardcoded default "." in glassfish-web.xml
 #       (which is relative to the domain root folder).
 #       Also, be aware that this props file cannot provide any value for lookups in glassfish-web.xml during servlet
 #       initialization, as this file will not have been read yet! The names and their values are in sync here and over
diff --git a/src/main/webapp/WEB-INF/glassfish-web.xml b/src/main/webapp/WEB-INF/glassfish-web.xml
index 9677bf089e2..015a309fd6b 100644
--- a/src/main/webapp/WEB-INF/glassfish-web.xml
+++ b/src/main/webapp/WEB-INF/glassfish-web.xml
@@ -11,13 +11,17 @@
   <parameter-encoding default-charset="UTF-8"/>
   <!-- Find a list of properties here: https://docs.oracle.com/cd/E19798-01/821-1750/beayb/index.html -->
   <!-- NOTE: relative paths given will be relative to com.sun.aas.instanceRoot (usually domain1 folder) -->
-  <property name="alternatedocroot_1" value="from=/guides/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
-  <property name="alternatedocroot_2" value="from=/dataexplore/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
-  <property name="alternatedocroot_logos" value="from=/logos/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
-  <property name="alternatedocroot_sitemap" value="from=/sitemap/* dir=${MPCONFIG=dataverse.files.docroot:${ENV=STORAGE_DIR:.}/docroot}"/>
+  <!-- NOTE: As we cannot use variables in defaults of ${MPCONFIG}, there is a workaround for containers necessary;
+             see src/main/docker/Dockerfile. Once Payara upstream fixes this, we can use STORAGE_DIR here. -->
+  <property name="alternatedocroot_1" value="from=/guides/* dir=${MPCONFIG=dataverse.files.docroot:./docroot}"/>
+  <property name="alternatedocroot_2" value="from=/dataexplore/* dir=${MPCONFIG=dataverse.files.docroot:./docroot}"/>
+  <property name="alternatedocroot_3" value="from=/logos/* dir=${MPCONFIG=dataverse.files.docroot:./docroot}"/>
+  <property name="alternatedocroot_4" value="from=/sitemap/* dir=${MPCONFIG=dataverse.files.docroot:./docroot}"/>
   <!--
     This folder is not only holding compiled JSP pages but also the place where file streams are stored
     during uploads. As Dataverse does not use any JSP, there will only be uploads stored here.
   -->
-  <property name="tempdir" value="${MPCONFIG=dataverse.files.uploads:${ENV=STORAGE_DIR:.}/uploads}"/>
+  <!-- NOTE: As we cannot use variables in defaults of ${MPCONFIG}, there is a workaround for containers necessary;
+             see src/main/docker/Dockerfile. Once Payara upstream fixes this, we can use STORAGE_DIR here. -->
+  <property name="tempdir" value="${MPCONFIG=dataverse.files.uploads:./uploads}"/>
 </glassfish-web-app>

From ec131f8b93463047811ff3b0ed626f61ee831041 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 16:15:28 +0200
Subject: [PATCH 305/922] fix(settings): make ConfigCheckService use
 Files.exist #9662

With Files.notExist if some folder does not have the "execute"
attribute, it cannot detect a folder does not exist.
Inverting the Files.exists call solves the problem.

Also adding tests for the business logic.
---
 .../settings/ConfigCheckService.java          |  2 +-
 .../settings/ConfigCheckServiceTest.java      | 92 +++++++++++++++++++
 2 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
index b175eafc3e0..83c3f6ac90d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
@@ -49,7 +49,7 @@ public boolean checkSystemDirectories() {
         
         boolean success = true;
         for (Path path : paths.keySet()) {
-            if (Files.notExists(path)) {
+            if (! Files.exists(path)) {
                 try {
                     Files.createDirectories(path);
                 } catch (IOException e) {
diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
new file mode 100644
index 00000000000..796448e579a
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
@@ -0,0 +1,92 @@
+package edu.harvard.iq.dataverse.settings;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Set;
+
+import static java.nio.file.attribute.PosixFilePermission.GROUP_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+
+class ConfigCheckServiceTest {
+
+    @Nested
+    class TestDirNotWritable {
+        @TempDir
+        Path testDir;
+        
+        private String oldUploadDirSetting;
+        
+        @BeforeEach
+        void setUp() throws IOException {
+            Files.setPosixFilePermissions(this.testDir, Set.of(OWNER_READ, GROUP_READ));
+            
+            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
+            //       @JvmSetting from static methods. Should be deleted.
+            this.oldUploadDirSetting = System.getProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey());
+            System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.testDir.toString());
+        }
+        
+        @AfterEach
+        void tearDown() {
+            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
+            //       @JvmSetting from static methods. Should be deleted.
+            if (this.oldUploadDirSetting != null)
+                System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.oldUploadDirSetting);
+        }
+        
+        @Test
+        void writeCheckFails() {
+            Assumptions.assumeTrue(Files.exists(this.testDir));
+            
+            ConfigCheckService sut = new ConfigCheckService();
+            Assertions.assertFalse(sut.checkSystemDirectories());
+        }
+    }
+    
+    @Nested
+    class TestDirNotExistent {
+        @TempDir
+        Path testDir;
+        String subFolder = "foobar";
+        
+        String oldUploadDirSetting;
+        
+        @BeforeEach
+        void setUp() throws IOException {
+            // Make test dir not writeable, so the subfolder cannot be created
+            Files.setPosixFilePermissions(this.testDir, Set.of(OWNER_READ, GROUP_READ));
+            
+            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
+            //       @JvmSetting from static methods. Should be deleted.
+            oldUploadDirSetting = System.getProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey());
+            System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.testDir.resolve(this.subFolder).toString());
+        }
+        
+        @AfterEach
+        void tearDown() {
+            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
+            //       @JvmSetting from static methods. Should be deleted.
+            if (this.oldUploadDirSetting != null)
+                System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.oldUploadDirSetting);
+        }
+        
+        @Test
+        void mkdirFails() {
+            Assumptions.assumeTrue(Files.exists(this.testDir));
+            Assumptions.assumeFalse(Files.exists(this.testDir.resolve(this.subFolder)));
+            
+            ConfigCheckService sut = new ConfigCheckService();
+            Assertions.assertFalse(sut.checkSystemDirectories());
+        }
+    }
+    
+}
\ No newline at end of file

From 4cd62eb6ed0812fec031e9328d3595dd13616225 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 22 Aug 2023 13:23:49 -0400
Subject: [PATCH 306/922] (ongoing experiments; a lot of these changes are
 temporary and will be deleted) #9763

---
 .../harvard/iq/dataverse/DatasetVersion.java  |  4 ++-
 .../dataverse/DatasetVersionServiceBean.java  | 31 +++++++++++++++++--
 .../harvard/iq/dataverse/api/Datasets.java    | 20 ++++++++++++
 .../command/impl/ListVersionsCommand.java     |  4 ++-
 .../search/SearchIncludeFragment.java         |  1 +
 .../iq/dataverse/util/json/JsonPrinter.java   |  5 +++
 6 files changed, 60 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
index 8d4dafad62a..f547f2963d1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java
@@ -72,7 +72,9 @@
     @NamedQuery(name = "DatasetVersion.findByDataset",
                 query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC"), 
     @NamedQuery(name = "DatasetVersion.findReleasedByDataset",
-                query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId AND o.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC")})
+                query = "SELECT o FROM DatasetVersion o WHERE o.dataset.id=:datasetId AND o.versionState=edu.harvard.iq.dataverse.DatasetVersion.VersionState.RELEASED ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC")/*,
+    @NamedQuery(name = "DatasetVersion.findVersionElements",
+                query = "SELECT o.id, o.versionState, o.versionNumber, o.minorVersionNumber FROM DatasetVersion o WHERE o.dataset.id=:datasetId ORDER BY o.versionNumber DESC, o.minorVersionNumber DESC")*/})
     
     
 @Entity
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 1edc281fa3e..fbed7d93cdd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -1301,7 +1301,7 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
                 .setParameter("datasetVersionId", datasetVersion.getId());
         
         if (limit == null && offset == null) {
-            query.setHint("eclipselink.left-join-fetch", "fm.dataFile.ingestRequest")
+            query = query.setHint("eclipselink.left-join-fetch", "fm.dataFile.ingestRequest")
                     .setHint("eclipselink.left-join-fetch", "fm.dataFile.thumbnailForDataset")
                     .setHint("eclipselink.left-join-fetch", "fm.dataFile.dataTables")
                     .setHint("eclipselink.left-join-fetch", "fm.fileCategories")
@@ -1312,10 +1312,35 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         } else {
             // @todo: is there really no way to use offset-limit with left join hints?
             if (limit != null) {
-                query.setMaxResults(limit);
+                query = query.setMaxResults(limit);
             }
             if (offset != null) {
-                query.setFirstResult(offset);
+                query = query.setFirstResult(offset);
+            }
+        }
+        return query.getResultList();
+    }
+    
+    public List<FileMetadata> getFileMetadatasByDbId(Long versionId, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
+        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
+                .setParameter("datasetVersionId", versionId);
+        
+        if (limit == null && offset == null) {
+            query = query.setHint("eclipselink.left-join-fetch", "fm.dataFile.ingestRequest")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.thumbnailForDataset")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.dataTables")
+                    .setHint("eclipselink.left-join-fetch", "fm.fileCategories")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.embargo")
+                    .setHint("eclipselink.left-join-fetch", "fm.datasetVersion")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.releaseUser")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.creator");
+        } else {
+            // @todo: is there really no way to use offset-limit with left join hints?
+            if (limit != null) {
+                query = query.setMaxResults(limit);
+            }
+            if (offset != null) {
+                query = query.setFirstResult(offset);
             }
         }
         return query.getResultList();
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 48755d4ea8a..47c249b7c8a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -467,6 +467,9 @@ public Response useDefaultCitationDate(@Context ContainerRequestContext crc, @Pa
     @AuthRequired
     @Path("{id}/versions")
     public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("includeFiles") Boolean includeFiles, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) {
+        // @todo:  when full versions list - including files - is requested, consider 
+        // using datasetservice.findDeep() (needs testing on "monstrous" datasets 
+        // with a lot of versions!)
         return response( req ->
              ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id), offset, limit) )
                                 .stream()
@@ -502,6 +505,23 @@ public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam
         }, getRequestUser(crc));
     }
     
+    //@todo: remember to delete this! (for experiments only!)
+    @GET
+    @AuthRequired
+    @Path("{id}/versions/{versionId}/files2")
+    public Response getVersionFiles2(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") Long versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response( req -> {
+            //DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            try {
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
+            }
+            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatasByDbId(versionId, limit, offset, fileMetadatasOrderCriteria)));
+        }, getRequestUser(crc));
+    }
+    
     @GET
     @AuthRequired
     @Path("{id}/dirindex")
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
index 80a5fe9b080..d3675a8f206 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
@@ -47,7 +47,9 @@ public List<DatasetVersion> execute(CommandContext ctxt) throws CommandException
         
         if (offset == null && limit == null) {
             // @todo: this fragment can be dropped, and the service-based method below 
-            // can be used for both cases. 
+            // can be used for both cases.
+            // @todo: on the other hand, consider using datasetservice.findDeep()
+            // when a full list of versions is requested. 
             List<DatasetVersion> outputList = new LinkedList<>();
             for (DatasetVersion dsv : ds.getVersions()) {
                 if (dsv.isReleased() || includeUnpublished) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index e249b81c983..5c5dc8b5171 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -205,6 +205,7 @@ public String searchRedirect(String dataverseRedirectPage, Dataverse dataverseIn
          */
         
         dataverse = dataverseIn;
+        logger.info("redirect page: "+dataverseRedirectPage);
         dataverseRedirectPage = StringUtils.isBlank(dataverseRedirectPage) ? "dataverse.xhtml" : dataverseRedirectPage;
         String optionalDataverseScope = "&alias=" + dataverse.getAlias();
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index dc8971c9539..68f0be3a067 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -373,6 +373,9 @@ public static JsonObjectBuilder json(DatasetVersion dsv, boolean includeFiles) {
     }
 
     public static JsonObjectBuilder json(DatasetVersion dsv, List<String> anonymizedFieldTypeNamesList, boolean includeFiles) {
+    /*    return json(dsv, null, includeFiles, null);
+    }
+    public static JsonObjectBuilder json(DatasetVersion dsv, List<String> anonymizedFieldTypeNamesList, boolean includeFiles, Long numberOfFiles) {*/
         Dataset dataset = dsv.getDataset();
         JsonObjectBuilder bld = jsonObjectBuilder()
                 .add("id", dsv.getId()).add("datasetId", dataset.getId())
@@ -388,6 +391,8 @@ public static JsonObjectBuilder json(DatasetVersion dsv, List<String> anonymized
                 .add("alternativePersistentId", dataset.getAlternativePersistentIdentifier())
                 .add("publicationDate", dataset.getPublicationDateFormattedYYYYMMDD())
                 .add("citationDate", dataset.getCitationDateFormattedYYYYMMDD());
+                //.add("numberOfFiles", numberOfFiles);
+        
         License license = DatasetUtil.getLicense(dsv);
         if (license != null) {
             bld.add("license", jsonLicense(dsv));

From 956f623b250b333956423cd3adbe2c7570df0f6f Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 22 Aug 2023 14:38:22 -0400
Subject: [PATCH 307/922] set build number #9814

---
 .github/workflows/deploy_beta_testing.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml
index f858272ca2b..0c0d5168906 100644
--- a/.github/workflows/deploy_beta_testing.yml
+++ b/.github/workflows/deploy_beta_testing.yml
@@ -22,6 +22,9 @@ jobs:
         working-directory: src/main/resources/META-INF
         run: echo -e "dataverse.feature.api-session-auth=true" >> microprofile-config.properties
 
+      - name: Set build number
+        run: scripts/installer/custom-build-number
+
       - name: Build application war
         run: mvn package
 

From 848f564a1c63656381352a6a0f52443bd05f9cb7 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 23:41:42 +0200
Subject: [PATCH 308/922] test(settings): make ConfigCheckService actually
 testable #9662

---
 .../settings/ConfigCheckServiceTest.java      | 82 ++++++++++---------
 .../META-INF/microprofile-config.properties   |  7 +-
 2 files changed, 50 insertions(+), 39 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
index 796448e579a..1018ad8d47b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
@@ -1,6 +1,6 @@
 package edu.harvard.iq.dataverse.settings;
 
-import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeEach;
@@ -17,35 +17,32 @@
 import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
 
 class ConfigCheckServiceTest {
+    
+    @TempDir
+    static Path testDir;
+    
+    private static final String testDirProp = "test.filesDir";
+    
+    @AfterAll
+    static void tearDown() {
+        System.clearProperty(testDirProp);
+    }
 
     @Nested
     class TestDirNotWritable {
-        @TempDir
-        Path testDir;
         
-        private String oldUploadDirSetting;
+        Path notWriteableSubfolder = testDir.resolve("readonly");
         
         @BeforeEach
         void setUp() throws IOException {
-            Files.setPosixFilePermissions(this.testDir, Set.of(OWNER_READ, GROUP_READ));
-            
-            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
-            //       @JvmSetting from static methods. Should be deleted.
-            this.oldUploadDirSetting = System.getProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey());
-            System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.testDir.toString());
-        }
-        
-        @AfterEach
-        void tearDown() {
-            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
-            //       @JvmSetting from static methods. Should be deleted.
-            if (this.oldUploadDirSetting != null)
-                System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.oldUploadDirSetting);
+            Files.createDirectory(notWriteableSubfolder);
+            Files.setPosixFilePermissions(notWriteableSubfolder, Set.of(OWNER_READ, GROUP_READ));
+            System.setProperty(testDirProp, notWriteableSubfolder.toString());
         }
         
         @Test
         void writeCheckFails() {
-            Assumptions.assumeTrue(Files.exists(this.testDir));
+            Assumptions.assumeTrue(Files.exists(notWriteableSubfolder));
             
             ConfigCheckService sut = new ConfigCheckService();
             Assertions.assertFalse(sut.checkSystemDirectories());
@@ -54,38 +51,47 @@ void writeCheckFails() {
     
     @Nested
     class TestDirNotExistent {
-        @TempDir
-        Path testDir;
-        String subFolder = "foobar";
         
-        String oldUploadDirSetting;
+        Path notExistTestfolder = testDir.resolve("parent-readonly");
+        Path notExistConfigSubfolder = notExistTestfolder.resolve("foobar");
         
         @BeforeEach
         void setUp() throws IOException {
+            Files.createDirectory(notExistTestfolder);
             // Make test dir not writeable, so the subfolder cannot be created
-            Files.setPosixFilePermissions(this.testDir, Set.of(OWNER_READ, GROUP_READ));
+            Files.setPosixFilePermissions(notExistTestfolder, Set.of(OWNER_READ, GROUP_READ));
+            System.setProperty(testDirProp, notExistConfigSubfolder.toString());
+        }
+        
+        @Test
+        void mkdirFails() {
+            Assumptions.assumeTrue(Files.exists(notExistTestfolder));
+            Assumptions.assumeFalse(Files.exists(notExistConfigSubfolder));
             
-            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
-            //       @JvmSetting from static methods. Should be deleted.
-            oldUploadDirSetting = System.getProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey());
-            System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.testDir.resolve(this.subFolder).toString());
+            ConfigCheckService sut = new ConfigCheckService();
+            Assertions.assertFalse(sut.checkSystemDirectories());
         }
+    }
+    
+    @Nested
+    class TestDirCreated {
+
+        Path missingToBeCreatedTestfolder = testDir.resolve("create-me");
+        Path missingToBeCreatedSubfolder = missingToBeCreatedTestfolder.resolve("foobar");
         
-        @AfterEach
-        void tearDown() {
-            // TODO: This is a workaround until PR #9273 is merged, providing the ability to lookup values for
-            //       @JvmSetting from static methods. Should be deleted.
-            if (this.oldUploadDirSetting != null)
-                System.setProperty(JvmSettings.UPLOADS_DIRECTORY.getScopedKey(), this.oldUploadDirSetting);
+        @BeforeEach
+        void setUp() throws IOException {
+            Files.createDirectory(missingToBeCreatedTestfolder);
+            System.setProperty(testDirProp, missingToBeCreatedSubfolder.toString());
         }
         
         @Test
-        void mkdirFails() {
-            Assumptions.assumeTrue(Files.exists(this.testDir));
-            Assumptions.assumeFalse(Files.exists(this.testDir.resolve(this.subFolder)));
+        void mkdirSucceeds() {
+            Assumptions.assumeTrue(Files.exists(missingToBeCreatedTestfolder));
+            Assumptions.assumeFalse(Files.exists(missingToBeCreatedSubfolder));
             
             ConfigCheckService sut = new ConfigCheckService();
-            Assertions.assertFalse(sut.checkSystemDirectories());
+            Assertions.assertTrue(sut.checkSystemDirectories());
         }
     }
     
diff --git a/src/test/resources/META-INF/microprofile-config.properties b/src/test/resources/META-INF/microprofile-config.properties
index 21f70b53896..8e5521f8287 100644
--- a/src/test/resources/META-INF/microprofile-config.properties
+++ b/src/test/resources/META-INF/microprofile-config.properties
@@ -8,4 +8,9 @@
 dataverse.pid.ezid.api-url=http://example.org
 # Also requires the username and the password to be present when used in production, use a default for unit testing.
 dataverse.pid.ezid.username=Dataverse Unit Test
-dataverse.pid.ezid.password=supersecret
\ No newline at end of file
+dataverse.pid.ezid.password=supersecret
+
+# To test ConfigCheckService, point our files directories to a common test dir
+dataverse.files.directory=${test.filesDir}
+dataverse.files.uploads=${test.filesDir}/uploads
+dataverse.files.docroot=${test.filesDir}/docroot

From c49ee0ab86891521801969a73367fd1cb50817ee Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 22 Aug 2023 23:47:45 +0200
Subject: [PATCH 309/922] test(settings): make ConfigCheckService require
 absolute paths #9662

---
 .../iq/dataverse/settings/ConfigCheckService.java      |  7 +++++++
 .../iq/dataverse/settings/ConfigCheckServiceTest.java  | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
index 83c3f6ac90d..a2c3f53d59d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/ConfigCheckService.java
@@ -49,6 +49,13 @@ public boolean checkSystemDirectories() {
         
         boolean success = true;
         for (Path path : paths.keySet()) {
+            // Check if the configured path is absolute - avoid potential problems with relative paths this way
+            if (! path.isAbsolute()) {
+                logger.log(Level.SEVERE, () -> "Configured directory " + path + " for " + paths.get(path) + " is not absolute");
+                success = false;
+                continue;
+            }
+            
             if (! Files.exists(path)) {
                 try {
                     Files.createDirectories(path);
diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
index 1018ad8d47b..b031b9429c6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
@@ -27,6 +27,16 @@ class ConfigCheckServiceTest {
     static void tearDown() {
         System.clearProperty(testDirProp);
     }
+    
+    @Nested
+    class TestDirNotAbsolute {
+        @Test
+        void nonAbsolutePathForTestDir() {
+            System.setProperty(testDirProp, "foobar");
+            ConfigCheckService sut = new ConfigCheckService();
+            Assertions.assertFalse(sut.checkSystemDirectories());
+        }
+    }
 
     @Nested
     class TestDirNotWritable {

From 28ddccc6d0f0baf4a3e48243b81c1ed001428b13 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 23 Aug 2023 10:18:27 +0200
Subject: [PATCH 310/922] fix(test,settings): make SiteMapUtilTest use
 test.filesDir property

This is also used in ConfigCheckServiceTest to verify the checks are
working. This property is picked up when sitemap util looks up the
storage location via MPCONFIG, parsing the default values during
testing from src/test/resources/META-INF/microprofile-config.properties
---
 .../edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java
index 4f2b00bbea4..2ded6cb6a33 100644
--- a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java
@@ -82,7 +82,8 @@ public void testUpdateSiteMap() throws IOException, ParseException {
         String tmpDir = tmpDirPath.toString();
         File docroot = new File(tmpDir + File.separator + "docroot");
         docroot.mkdirs();
-        System.setProperty("com.sun.aas.instanceRoot", tmpDir);
+        // TODO: this and the above should be replaced with JUnit 5 @TestDir
+        System.setProperty("test.filesDir", tmpDir);
 
         SiteMapUtil.updateSiteMap(dataverses, datasets);
 
@@ -117,7 +118,7 @@ public void testUpdateSiteMap() throws IOException, ParseException {
         assertFalse(sitemapString.contains(harvestedPid));
         assertFalse(sitemapString.contains(deaccessionedPid));
 
-        System.clearProperty("com.sun.aas.instanceRoot");
+        System.clearProperty("test.filesDir");
 
     }
 

From ba8e6d221ad9e0ce5de4391e96757d120e0313b4 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 23 Aug 2023 10:20:29 +0200
Subject: [PATCH 311/922] fix(test,settings): provide default for test.filesDir

By providing a sane default unter /tmp, we enable a few tests that
do not use a custom testclass scoped directory to run
---
 src/test/resources/META-INF/microprofile-config.properties | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/test/resources/META-INF/microprofile-config.properties b/src/test/resources/META-INF/microprofile-config.properties
index 8e5521f8287..113a098a1fe 100644
--- a/src/test/resources/META-INF/microprofile-config.properties
+++ b/src/test/resources/META-INF/microprofile-config.properties
@@ -10,7 +10,9 @@ dataverse.pid.ezid.api-url=http://example.org
 dataverse.pid.ezid.username=Dataverse Unit Test
 dataverse.pid.ezid.password=supersecret
 
-# To test ConfigCheckService, point our files directories to a common test dir
+# To test ConfigCheckService, point our files directories to a common test dir by overriding the
+# property test.filesDir via system properties
+test.filesDir=/tmp/dataverse
 dataverse.files.directory=${test.filesDir}
 dataverse.files.uploads=${test.filesDir}/uploads
 dataverse.files.docroot=${test.filesDir}/docroot

From 9aafbfce4eeb086a2b1b1f7fee39adff2d839624 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 23 Aug 2023 10:03:23 +0100
Subject: [PATCH 312/922] Added: release notes and docs

---
 doc/release-notes/9785-files-api-extension-search-text.md | 3 +++
 doc/sphinx-guides/source/api/native-api.rst               | 8 ++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 doc/release-notes/9785-files-api-extension-search-text.md

diff --git a/doc/release-notes/9785-files-api-extension-search-text.md b/doc/release-notes/9785-files-api-extension-search-text.md
new file mode 100644
index 00000000000..fb185e1c7af
--- /dev/null
+++ b/doc/release-notes/9785-files-api-extension-search-text.md
@@ -0,0 +1,3 @@
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by search text through the `searchText` query parameter.
+
+The search will be applied to the labels and descriptions of the dataset files.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 3482df2816c..96555b2ba5f 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -980,6 +980,14 @@ Usage example:
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?contentType=image/png"
 
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the files that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?searchText=word"
+
 File access filtering is also optionally supported. In particular, by the following possible values:
 
 * ``Public``

From 396ffff1c5dc2b38435c140ba5860fe5fbee9fd6 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 23 Aug 2023 11:45:49 +0200
Subject: [PATCH 313/922] style(settings): unify default dataverse.files dir
 options

- No more profile to work around Payaras bug with overriding profiled
  values
- Group together because every item using $STORAGE_DIR and a default
  to match classic installs now
---
 src/main/resources/META-INF/microprofile-config.properties | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 11471663fc3..f51a728332e 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -9,15 +9,13 @@ dataverse.build=
 %ct.dataverse.siteUrl=http://${dataverse.fqdn}:8080
 
 # FILES
-dataverse.files.directory=/tmp/dataverse
-# The variables are replaced with the environment variables from our base image, but still easy to override
-%ct.dataverse.files.directory=${STORAGE_DIR}
-# NOTE: The following uses STORAGE_DIR for both containers and classic installations. By defaulting to
+# NOTE: The following uses STORAGE_DIR for both containers and classic installations. When defaulting to
 #       "com.sun.aas.instanceRoot" if not present, it equals the hardcoded default "." in glassfish-web.xml
 #       (which is relative to the domain root folder).
 #       Also, be aware that this props file cannot provide any value for lookups in glassfish-web.xml during servlet
 #       initialization, as this file will not have been read yet! The names and their values are in sync here and over
 #       there to ensure the config checker is able to check for the directories (exist + writeable).
+dataverse.files.directory=${STORAGE_DIR:/tmp/dataverse}
 dataverse.files.uploads=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/uploads
 dataverse.files.docroot=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/docroot
 

From d37eedfd7898eecb9fecb8bbe564f8efd76d5e8b Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 23 Aug 2023 11:48:26 +0200
Subject: [PATCH 314/922] docs(settings): refactor docs on the important
 directories and how to configure them #9662

---
 .../source/installation/config.rst            | 49 ++++++++++++++-----
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f9fe74afc7c..dc31b1afae8 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1759,8 +1759,8 @@ protocol, host, and port number and should not include a trailing slash.
 dataverse.files.directory
 +++++++++++++++++++++++++
 
-Please provide an absolute path to a directory backed by some mounted file system. This directory is used for a number
-of purposes:
+Providing an explicit location here makes it easier to reuse some mounted filesystem and we recommend doing so
+to avoid filled up disks, aid in performance, etc. This directory is used for a number of purposes:
 
 1. ``<dataverse.files.directory>/temp`` after uploading, data is temporarily stored here for ingest and/or before
    shipping to the final storage destination.
@@ -1773,24 +1773,51 @@ of purposes:
    under certain conditions. This directory may also be used by file stores for :ref:`permanent file storage <storage-files-dir>`,
    but this is controlled by other, store-specific settings.
 
-Defaults to ``/tmp/dataverse``. Can also be set via *MicroProfile Config API* sources, e.g. the environment variable
-``DATAVERSE_FILES_DIRECTORY``. Defaults to ``${STORAGE_DIR}`` for profile ``ct``, important for the
-:ref:`Dataverse Application Image <app-locations>`.
+Notes:
+
+- Please provide an absolute path to a directory backed by some mounted file system.
+- Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_DIRECTORY``.
+- Defaults to ``/tmp/dataverse`` in a :doc:`default installation <installation-main>`.
+- Defaults to ``${STORAGE_DIR}`` using our :ref:`Dataverse container <app-locations>` (resolving to ``/dv``).
+- During startup, this directory will be checked for existence and write access. It will be created for you
+  if missing. If it cannot be created or does not have proper write access, application deployment will fail.
 
 .. _dataverse.files.uploads:
 
 dataverse.files.uploads
 +++++++++++++++++++++++
 
-Configure a folder to store the incoming file stream during uploads (before transfering to `${dataverse.files.directory}/temp`).
+Configure a folder to store the incoming file stream during uploads (before transfering to ``${dataverse.files.directory}/temp``).
+Providing an explicit location here makes it easier to reuse some mounted filesystem.
 Please also see :ref:`temporary-file-storage` for more details.
-You can use an absolute path or a relative, which is relative to the application server domain directory.
 
-Defaults to ``./uploads``, which resolves to ``/usr/local/payara6/glassfish/domains/domain1/uploads`` in a default
-installation.
+Notes:
+
+- Please provide an absolute path to a directory backed by some mounted file system.
+- Defaults to ``${com.sun.aas.instanceRoot}/uploads`` in a :doc:`default installation <installation-main>`
+  (resolving to ``/usr/local/payara6/glassfish/domains/domain1/uploads``).
+- Defaults to ``${STORAGE_DIR}/uploads`` using our :ref:`Dataverse container <app-locations>` (resolving to ``/dv/uploads``).
+- Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_UPLOADS``.
+- During startup, this directory will be checked for existence and write access. It will be created for you
+  if missing. If it cannot be created or does not have proper write access, application deployment will fail.
+
+.. _dataverse.files.docroot:
+
+dataverse.files.docroot
++++++++++++++++++++++++
+
+Configure a folder to store and retrieve additional materials like user uploaded collection logos, generated sitemaps,
+and so on. Providing an explicit location here makes it easier to reuse some mounted filesystem.
+See also logo customization above.
+
+Notes:
 
-Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_UPLOADS``.
-Defaults to ``${STORAGE_DIR}/uploads`` for profile ``ct``, important for the :ref:`Dataverse Application Image <app-locations>`.
+- Defaults to ``${com.sun.aas.instanceRoot}/docroot`` in a :doc:`default installation <installation-main>`
+  (resolves to ``/usr/local/payara6/glassfish/domains/domain1/docroot``).
+- Defaults to ``${STORAGE_DIR}/docroot`` using our :ref:`Dataverse container <app-locations>` (resolving to ``/dv/docroot``).
+- Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_DOCROOT``.
+- During startup, this directory will be checked for existence and write access. It will be created for you
+  if missing. If it cannot be created or does not have proper write access, application deployment will fail.
 
 dataverse.auth.password-reset-timeout-in-minutes
 ++++++++++++++++++++++++++++++++++++++++++++++++

From a98395a167d6f1350a7684553517d400e28a8859 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Wed, 23 Aug 2023 17:50:27 +0200
Subject: [PATCH 315/922] Update HTTP to HTTPS URLs in classic-dev-env.rst

---
 doc/sphinx-guides/source/developers/classic-dev-env.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst
index 6feca558267..363fc87ae17 100755
--- a/doc/sphinx-guides/source/developers/classic-dev-env.rst
+++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst
@@ -46,7 +46,7 @@ On Linux, you are welcome to use the OpenJDK available from package managers.
 Install Netbeans or Maven
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-NetBeans IDE is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven.
+NetBeans IDE is recommended, and can be downloaded from https://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven.
 
 Below we describe how to build the Dataverse Software war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section.
 
@@ -86,7 +86,7 @@ On Mac, run this command:
 
 ``brew install jq``
 
-On Linux, install ``jq`` from your package manager or download a binary from http://stedolan.github.io/jq/
+On Linux, install ``jq`` from your package manager or download a binary from https://stedolan.github.io/jq/
 
 Install Payara
 ~~~~~~~~~~~~~~
@@ -134,7 +134,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager
 Install Solr
 ^^^^^^^^^^^^
 
-`Solr <http://lucene.apache.org/solr/>`_ 8.11.1 is required.
+`Solr <https://lucene.apache.org/solr/>`_ 8.11.1 is required.
 
 To install Solr, execute the following commands:
 
@@ -144,7 +144,7 @@ To install Solr, execute the following commands:
 
 ``cd /usr/local/solr``
 
-``curl -O http://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
+``curl -O https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
 
 ``tar xvfz solr-8.11.1.tgz``
 

From 4677cff58b63bdf5b3f0439b8f62ea01dcc91a90 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Thu, 24 Aug 2023 10:18:21 +0200
Subject: [PATCH 316/922] Use HTTPS URLs in README.md

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index d40e5f228f7..01de5b2c854 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 Dataverse&#174;  
 ===============
 
-Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).
+Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](https://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]).
 
 [dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more.
 
@@ -26,15 +26,15 @@ Dataverse is a trademark of President and Fellows of Harvard College and is regi
 [dataverse.org]: https://dataverse.org
 [demo.dataverse.org]: https://demo.dataverse.org
 [Dataverse community]: https://dataverse.org/developers
-[Installation Guide]: http://guides.dataverse.org/en/latest/installation/index.html
+[Installation Guide]: https://guides.dataverse.org/en/latest/installation/index.html
 [latest release]: https://github.com/IQSS/dataverse/releases
 [features]: https://dataverse.org/software-features
 [roadmap]: https://www.iq.harvard.edu/roadmap-dataverse-project
 [integrations]: https://dataverse.org/integrations
-[REST APIs]: http://guides.dataverse.org/en/latest/api/index.html
+[REST APIs]: https://guides.dataverse.org/en/latest/api/index.html
 [Contributing Guide]: CONTRIBUTING.md
 [mailing list]: https://groups.google.com/group/dataverse-community
 [community call]: https://dataverse.org/community-calls
-[chat.dataverse.org]: http://chat.dataverse.org
+[chat.dataverse.org]: https://chat.dataverse.org
 [Dataverse Community Meeting]: https://dataverse.org/events
 [open source]: LICENSE.md

From 4e6d948d712da42862b9f429d8ef65086a71baab Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 25 Aug 2023 10:00:30 -0400
Subject: [PATCH 317/922] remove req. that app and DV are on same host, note
 future todo

---
 .../java/edu/harvard/iq/dataverse/api/Datasets.java   | 11 +++--------
 .../iq/dataverse/globus/GlobusServiceBean.java        |  2 ++
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index dcd7eacf50b..b8165f0314f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3329,8 +3329,7 @@ public Response getTimestamps(@Context ContainerRequestContext crc, @PathParam("
     public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
                                             @PathParam("id") String datasetId,
                                             @FormDataParam("jsonData") String jsonData,
-                                            @Context UriInfo uriInfo,
-                                            @Context HttpHeaders headers
+                                            @Context UriInfo uriInfo
     ) throws IOException, ExecutionException, InterruptedException {
 
         logger.info(" ====  (api addGlobusFilesToDataset) jsonData   ====== " + jsonData);
@@ -3390,12 +3389,8 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
         }
 
 
-        String requestUrl = headers.getRequestHeader("origin").get(0);
-
-        if(requestUrl.contains("localhost")){
-            requestUrl = "http://localhost:8080";
-        }
-
+        String requestUrl = SystemConfig.getDataverseSiteUrlStatic();
+        
         // Async Call
         globusService.globusUpload(jsonData, token, dataset, requestUrl, authUser);
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index c2137dd1f47..5c387710844 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -737,6 +737,8 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
                             + datasetIdentifier + " -F jsonData='" + newjsonData + "'";
                     System.out.println("*******====command ==== " + command);
 
+                    //ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of calling API
+                
                     String output = addFilesAsync(command, globusLogger);
                     if (output.equalsIgnoreCase("ok")) {
                         // if(!taskSkippedFiles)

From a90aa22095b41a07cc9f9f11d51425dc940c5438 Mon Sep 17 00:00:00 2001
From: lubitchv <victoria.lubitch@utoronto.ca>
Date: Fri, 25 Aug 2023 11:50:38 -0400
Subject: [PATCH 318/922] alt title multiple

---
 scripts/api/data/dataset-create-new-all-default-fields.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/api/data/dataset-create-new-all-default-fields.json b/scripts/api/data/dataset-create-new-all-default-fields.json
index 4af128955c9..1118ed98a03 100644
--- a/scripts/api/data/dataset-create-new-all-default-fields.json
+++ b/scripts/api/data/dataset-create-new-all-default-fields.json
@@ -22,9 +22,9 @@
           },
           {
             "typeName": "alternativeTitle",
-            "multiple": false,
+            "multiple": true,
             "typeClass": "primitive",
-            "value": "Alternative Title"
+            "value": ["Alternative Title"]
           },
           {
             "typeName": "alternativeURL",

From 7dd17c732233ef63f808a0406675e2085cf0e666 Mon Sep 17 00:00:00 2001
From: Pradyumna Sridhara <95268690+prsridha@users.noreply.github.com>
Date: Sat, 26 Aug 2023 20:21:29 +0530
Subject: [PATCH 319/922] Added additional line in Permalinks config

Added an additional line to restart Payara after changing settings in Permalinks section
---
 doc/sphinx-guides/source/installation/config.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f9fe74afc7c..c9779c6fb05 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -305,6 +305,8 @@ Here are the configuration options for PermaLinks:
 - :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional)
 - :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to false)
 
+You must restart Payara after making changes to these settings.
+
 .. _auth-modes:
 
 Auth Modes: Local vs. Remote vs. Both

From e9cf04178e8a492bc1f1bda7f3113dc1364f6f59 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 28 Aug 2023 18:45:59 +0100
Subject: [PATCH 320/922] Stash: getVersionFileCounts endpoint WIP. Pending
 access type count, test coverage and refactor

---
 .../DatasetVersionFilesServiceBean.java       | 66 +++++++++++++++++--
 .../harvard/iq/dataverse/api/Datasets.java    | 34 +++++++++-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 47 ++++++++++---
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 ++
 4 files changed, 138 insertions(+), 15 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 751da6daba8..364ac36652c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -5,6 +5,7 @@
 import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
 
+import com.querydsl.core.Tuple;
 import com.querydsl.core.types.dsl.BooleanExpression;
 import com.querydsl.core.types.dsl.CaseBuilder;
 import com.querydsl.core.types.dsl.DateExpression;
@@ -21,7 +22,9 @@
 import java.io.Serializable;
 import java.sql.Timestamp;
 import java.time.LocalDate;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 @Stateless
 @Named
@@ -48,6 +51,57 @@ public enum DataFileAccessStatus {
         Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
     }
 
+    /**
+     * Given a DatasetVersion, returns its total file metadata count
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return long value of total file metadata count
+     */
+    public long getFileMetadataCount(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).stream().count();
+    }
+
+    /**
+     * Given a DatasetVersion, returns its file metadata count per content type
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return Map<String, Long> of file metadata counts per content type
+     */
+    public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        List<Tuple> contentTypeOccurrences = queryFactory
+                .select(fileMetadata.dataFile.contentType, fileMetadata.count())
+                .from(fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                .groupBy(fileMetadata.dataFile.contentType).fetch();
+        Map<String, Long> result = new HashMap<>();
+        for (Tuple occurrence : contentTypeOccurrences) {
+            result.put(occurrence.get(fileMetadata.dataFile.contentType), occurrence.get(fileMetadata.count()));
+        }
+        return result;
+    }
+
+    /**
+     * Given a DatasetVersion, returns its file metadata count per category name
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return Map<String, Long> of file metadata counts per category name
+     */
+    public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        List<Tuple> categoryNameOccurrences = queryFactory
+                .select(dataFileCategory.name, fileMetadata.count())
+                .from(dataFileCategory, fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.fileCategories.contains(dataFileCategory)))
+                .groupBy(dataFileCategory.name).fetch();
+        Map<String, Long> result = new HashMap<>();
+        for (Tuple occurrence : categoryNameOccurrences) {
+            result.put(occurrence.get(dataFileCategory.name), occurrence.get(fileMetadata.count()));
+        }
+        return result;
+    }
+
     /**
      * Returns a FileMetadata list of files in the specified DatasetVersion
      *
@@ -62,13 +116,13 @@ public enum DataFileAccessStatus {
      * @return a FileMetadata list from the specified DatasetVersion
      */
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> baseQuery = createBaseQuery(datasetVersion, orderCriteria);
+        JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
 
         if (contentType != null) {
             baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
         }
         if (accessStatus != null) {
-            baseQuery.where(createAccessStatusExpression(accessStatus));
+            baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
         }
         if (categoryName != null) {
             baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
@@ -78,7 +132,7 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
             baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
         }
 
-        applyOrderCriteriaToQuery(baseQuery, orderCriteria);
+        applyOrderCriteriaToGetFileMetadatasQuery(baseQuery, orderCriteria);
 
         if (limit != null) {
             baseQuery.limit(limit);
@@ -90,7 +144,7 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         return baseQuery.fetch();
     }
 
-    private JPAQuery<FileMetadata> createBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
+    private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
         JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
         if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) {
@@ -99,7 +153,7 @@ private JPAQuery<FileMetadata> createBaseQuery(DatasetVersion datasetVersion, Fi
         return baseQuery;
     }
 
-    private BooleanExpression createAccessStatusExpression(DataFileAccessStatus accessStatus) {
+    private BooleanExpression createGetFileMetadatasAccessStatusExpression(DataFileAccessStatus accessStatus) {
         QEmbargo embargo = fileMetadata.dataFile.embargo;
         BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
         BooleanExpression inactivelyEmbargoedExpression = embargo.isNull();
@@ -123,7 +177,7 @@ private BooleanExpression createAccessStatusExpression(DataFileAccessStatus acce
         return accessStatusExpression;
     }
 
-    private void applyOrderCriteriaToQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
+    private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
         DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
         switch (orderCriteria) {
             case NameZA:
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index dc192281695..297cffaa06f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -520,7 +520,39 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, searchText, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
-    
+
+    // TODO: Finalize and refine
+    @GET
+    @AuthRequired
+    @Path("{id}/versions/{versionId}/files/counts")
+    public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response(req -> {
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+
+            jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
+
+            Map<String, Long> fileMetadataCountsPerContentType = datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion);
+            JsonObjectBuilder fileMetadataCountsPerContentTypeJsonBuilder = Json.createObjectBuilder();
+            for (Map.Entry<String, Long> fileMetadataCount : fileMetadataCountsPerContentType.entrySet()) {
+                fileMetadataCountsPerContentTypeJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue());
+            }
+            jsonObjectBuilder.add("perContentType", fileMetadataCountsPerContentTypeJsonBuilder);
+
+            // TODO
+            jsonObjectBuilder.add("perAccessStatus", 3);
+
+            Map<String, Long> fileMetadataCountsPerCategoryName = datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion);
+            JsonObjectBuilder fileMetadataCountsPerCategoryNameJsonBuilder = Json.createObjectBuilder();
+            for (Map.Entry<String, Long> fileMetadataCount : fileMetadataCountsPerCategoryName.entrySet()) {
+                fileMetadataCountsPerCategoryNameJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue());
+            }
+            jsonObjectBuilder.add("perCategoryName", fileMetadataCountsPerCategoryNameJsonBuilder);
+
+            return ok(jsonObjectBuilder);
+        }, getRequestUser(crc));
+    }
+
     @GET
     @AuthRequired
     @Path("{id}/dirindex")
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index c346228d034..efd4c31df57 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -11,6 +11,7 @@
 
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.util.*;
 import java.util.logging.Logger;
 
 import org.junit.jupiter.api.AfterAll;
@@ -19,8 +20,6 @@
 import org.skyscreamer.jsonassert.JSONAssert;
 import org.junit.jupiter.api.Disabled;
 
-import java.util.List;
-import java.util.Map;
 import jakarta.json.JsonObject;
 
 import static jakarta.ws.rs.core.Response.Status.CREATED;
@@ -41,8 +40,6 @@
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
-import java.util.UUID;
-
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 
@@ -69,8 +66,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.HashMap;
+
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObjectBuilder;
@@ -3327,8 +3323,6 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
 
-        String testFileId3 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
-
         // Test page 3 (last)
         getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, apiToken);
 
@@ -3490,4 +3484,41 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
     }
+
+    @Test
+    public void getVersionFileCounts() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        String testFileName1 = "test_1.txt";
+        String testFileName2 = "test_2.txt";
+        String testFileName3 = "test_3.png";
+
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[50], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken);
+
+        String testDatasetVersion = ":latest";
+
+        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        LinkedHashMap<String, Integer> responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+
+        assertEquals((Integer) responseJsonPath.get("data.total"), 3);
+        assertEquals(responseCountPerContentTypeMap.get("image/png"), 1);
+        assertEquals(responseCountPerContentTypeMap.get("text/plain"), 2);
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 7939fd7d88d..69549f2e4a8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3341,4 +3341,10 @@ static Response createFileEmbargo(Integer datasetId, Integer fileId, String date
                 .urlEncodingEnabled(false)
                 .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
     }
+
+    static Response getVersionFileCounts(Integer datasetId, String version, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
+    }
 }

From cc5a1bf367105aade087add7bd442e2c9f9dd6e0 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 29 Aug 2023 10:43:33 +0100
Subject: [PATCH 321/922] Added: setFileCategories API endpoint

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 38 ++++++++++++----
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 44 +++++++++++++++++--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 14 ++++++
 3 files changed, 85 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 4c411a631f1..6712b68a09b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -53,6 +53,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -62,15 +63,12 @@
 import jakarta.ejb.EJBException;
 import jakarta.inject.Inject;
 import jakarta.json.Json;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonString;
+import jakarta.json.JsonValue;
+import jakarta.json.stream.JsonParsingException;
 import jakarta.servlet.http.HttpServletResponse;
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.*;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.HttpHeaders;
@@ -848,4 +846,28 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, @PathPar
             return ok(jsonDT(dataFile.getDataTables()));
         }, getRequestUser(crc));
     }
+
+    @POST
+    @AuthRequired
+    @Path("{id}/metadata/categories")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response setFileCategories(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            jakarta.json.JsonObject jsonObject;
+            try (StringReader stringReader = new StringReader(jsonBody)) {
+                jsonObject = Json.createReader(stringReader).readObject();
+                JsonArray requestedCategoriesJson = jsonObject.getJsonArray("categories");
+                FileMetadata fileMetadata = dataFile.getFileMetadata();
+                for (JsonValue jsonValue : requestedCategoriesJson) {
+                    JsonString jsonString = (JsonString) jsonValue;
+                    fileMetadata.addCategoryByName(jsonString.getString());
+                }
+                execCommand(new UpdateDatasetVersionCommand(fileMetadata.getDataFile().getOwner(), req));
+                return ok("Categories of file " + dataFileId + " updated.");
+            } catch (JsonParsingException jpe) {
+                return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage());
+            }
+        }, getRequestUser(crc));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 77519efa2cc..1004d3812d5 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2,6 +2,8 @@
 
 import io.restassured.RestAssured;
 import io.restassured.response.Response;
+
+import java.util.List;
 import java.util.logging.Logger;
 
 import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism;
@@ -30,11 +32,10 @@
 
 import static jakarta.ws.rs.core.Response.Status.*;
 import org.hamcrest.CoreMatchers;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.startsWith;
-import static org.hamcrest.CoreMatchers.nullValue;
 import org.hamcrest.Matchers;
 
+import static org.hamcrest.CoreMatchers.*;
+import static org.hamcrest.CoreMatchers.hasItem;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -2184,4 +2185,41 @@ public void testGetFileDataTables() throws InterruptedException {
         int dataTablesNumber = JsonPath.from(getFileDataTablesForTabularFileResponse.body().asString()).getList("data").size();
         assertTrue(dataTablesNumber > 0);
     }
+
+    @Test
+    public void testSetFileCategories() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Set categories
+        String testCategory1 = "testCategory1";
+        String testCategory2 = "testCategory2";
+        List<String> testCategories = List.of(testCategory1, testCategory2);
+        Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, testCategories);
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get file data and check for new categories
+        Response getFileDataResponse = UtilIT.getFileData(dataFileId, apiToken);
+        getFileDataResponse.prettyPrint();
+        getFileDataResponse.then().assertThat()
+                .body("data.categories", hasItem(testCategory1))
+                .body("data.categories", hasItem(testCategory2))
+                .statusCode(OK.getStatusCode());
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 69549f2e4a8..0fcb7f4f287 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3347,4 +3347,18 @@ static Response getVersionFileCounts(Integer datasetId, String version, String a
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
     }
+
+    static Response setFileCategories(String dataFileId, String apiToken, List<String> categories) {
+        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
+        for (String category : categories) {
+            jsonArrayBuilder.add(category);
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("categories", jsonArrayBuilder);
+        String jsonString = jsonObjectBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .post("/api/files/" + dataFileId + "/metadata/categories");
+    }
 }

From eadab48a4215b61198255f2ce93bc5088ae0e4d8 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 29 Aug 2023 10:51:19 +0100
Subject: [PATCH 322/922] Fixed: getVersionFilesIT test case for category
 filtering

---
 .../edu/harvard/iq/dataverse/api/DatasetsIT.java   | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index efd4c31df57..5d4fad496d6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3406,13 +3406,21 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Category Name
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, "nonExistentCategory", null, null, apiToken);
+        String testCategory = "testCategory";
+        Response setFileCategoriesResponse = UtilIT.setFileCategories(testFileId1, apiToken, List.of(testCategory));
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+        setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory));
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, apiToken);
 
         getVersionFilesResponseCategoryName.then().assertThat()
-                .statusCode(OK.getStatusCode());
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1))
+                .body("data[1].label", equalTo(testFileName2));
 
         fileMetadatasCount = getVersionFilesResponseCategoryName.jsonPath().getList("data").size();
-        assertEquals(0, fileMetadatasCount);
+        assertEquals(2, fileMetadatasCount);
 
         // Test Access Status Restricted
         Response restrictFileResponse = UtilIT.restrictFile(String.valueOf(testFileId1), true, apiToken);

From 70b9193ff750e119b348f14ce938255cd7a8b497 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 29 Aug 2023 11:12:09 +0100
Subject: [PATCH 323/922] Added: getVersionFileCounts count per category test
 coverage

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 5d4fad496d6..1b3811a6d66 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3506,8 +3506,9 @@ public void getVersionFileCounts() throws IOException {
         Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
-        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
+        // Creating test files
         String testFileName1 = "test_1.txt";
         String testFileName2 = "test_2.txt";
         String testFileName3 = "test_3.png";
@@ -3516,17 +3517,26 @@ public void getVersionFileCounts() throws IOException {
         UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken);
         UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken);
 
-        String testDatasetVersion = ":latest";
+        // Creating a categorized test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+        String testCategory = "testCategory";
+        UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory));
 
-        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, apiToken);
+        // Getting the file counts and assert each count
+        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken);
 
         getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath();
         LinkedHashMap<String, Integer> responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        LinkedHashMap<String, Integer> responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
 
-        assertEquals((Integer) responseJsonPath.get("data.total"), 3);
-        assertEquals(responseCountPerContentTypeMap.get("image/png"), 1);
+        assertEquals((Integer) responseJsonPath.get("data.total"), 4);
+        assertEquals(responseCountPerContentTypeMap.get("image/png"), 2);
         assertEquals(responseCountPerContentTypeMap.get("text/plain"), 2);
+        assertEquals(responseCountPerCategoryNameMap.get(testCategory), 1);
     }
 }

From b5e47b98a08f25c1160fc651b84bc1fbefe3dfa4 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 29 Aug 2023 12:52:33 -0400
Subject: [PATCH 324/922] fix retrieveSize parsing, refactoring

---
 .../dataaccess/GlobusOverlayAccessIO.java     | 169 ++++++------------
 .../dataaccess/RemoteOverlayAccessIO.java     |  82 ++++-----
 2 files changed, 93 insertions(+), 158 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 081c5a622aa..6a22f8b68f3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -4,14 +4,12 @@
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
-import edu.harvard.iq.dataverse.datavariable.DataVariable;
 import edu.harvard.iq.dataverse.globus.AccessToken;
 import edu.harvard.iq.dataverse.globus.GlobusServiceBean;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -56,7 +54,6 @@
 
 /**
  * @author qqmyers
- * @param <T> what it stores
  */
 /*
  * Globus Overlay Driver
@@ -64,14 +61,13 @@
  * StorageIdentifier format: <globusDriverId>://<local
  * id>//<globusEndpointIdentifier>/<absolutePath>
  */
-public class GlobusOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
+public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
     private StorageIO<DvObject> baseStore = null;
     private String path = null;
     private String endpointWithBasePath = null;
-    private String globusToken = null;
 
     private static HttpClientContext localContext = HttpClientContext.create();
     private PoolingHttpClientConnectionManager cm = null;
@@ -117,103 +113,37 @@ private void validatePath(String relPath) throws IOException {
         }
     }
 
-    @Override
-    public void open(DataAccessOption... options) throws IOException {
-
-        baseStore.open(options);
-
-        DataAccessRequest req = this.getRequest();
-
-        if (isWriteAccessRequested(options)) {
-            isWriteAccess = true;
-            isReadAccess = false;
-        } else {
-            isWriteAccess = false;
-            isReadAccess = true;
-        }
-
-        if (dvObject instanceof DataFile) {
-            String storageIdentifier = dvObject.getStorageIdentifier();
-
-            DataFile dataFile = this.getDataFile();
-
-            if (req != null && req.getParameter("noVarHeader") != null) {
-                baseStore.setNoVarHeader(true);
-            }
-
-            if (storageIdentifier == null || "".equals(storageIdentifier)) {
-                throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile.");
-            }
-
-            // Fix new DataFiles: DataFiles that have not yet been saved may use this method
-            // when they don't have their storageidentifier in the final form
-            // So we fix it up here. ToDo: refactor so that storageidentifier is generated
-            // by the appropriate StorageIO class and is final from the start.
-            logger.fine("StorageIdentifier is: " + storageIdentifier);
-
-            if (isReadAccess) {
-                if (dataFile.getFilesize() >= 0) {
-                    this.setSize(dataFile.getFilesize());
-                } else {
-                    logger.fine("Setting size");
-                    this.setSize(getSizeFromGlobus());
-                }
-                if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values")
-                        && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
-
-                    List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
-                    String varHeaderLine = generateVariableHeader(datavariables);
-                    this.setVarHeader(varHeaderLine);
-                }
-
-            }
-
-            this.setMimeType(dataFile.getContentType());
-
-            try {
-                this.setFileName(dataFile.getFileMetadata().getLabel());
-            } catch (Exception ex) {
-                this.setFileName("unknown");
-            }
-        } else if (dvObject instanceof Dataset) {
-            throw new IOException(
-                    "Data Access: RemoteOverlay Storage driver does not support dvObject type Dataverse yet");
-        } else if (dvObject instanceof Dataverse) {
-            throw new IOException(
-                    "Data Access: RemoteOverlay Storage driver does not support dvObject type Dataverse yet");
-        } else {
-            this.setSize(getSizeFromGlobus());
-        }
-    }
-
     // Call the Globus API to get the file size
-    private long getSizeFromGlobus() {
+    @Override
+    long retrieveSize() {
         // Construct Globus URL
         URI absoluteURI = null;
         try {
             int filenameStart = path.lastIndexOf("/") + 1;
             int pathStart = endpointWithBasePath.indexOf("/");
-logger.info("endpointWithBasePath: " + endpointWithBasePath);
+            logger.info("endpointWithBasePath: " + endpointWithBasePath);
             String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "")
                     + path.substring(0, filenameStart);
             logger.info("directoryPath: " + directoryPath);
             String filename = path.substring(filenameStart);
             String endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart - 1) : endpointWithBasePath;
 
-            absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint + "/ls?path=" + directoryPath + "&filter=name:" + filename);
+            absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint
+                    + "/ls?path=" + directoryPath + "&filter=name:" + filename);
             HttpGet get = new HttpGet(absoluteURI);
-            
+
             logger.info("Token is " + globusAccessToken);
             get.addHeader("Authorization", "Bearer " + globusAccessToken);
             CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
             if (response.getStatusLine().getStatusCode() == 200) {
-                //Get reponse as string
+                // Get reponse as string
                 String responseString = EntityUtils.toString(response.getEntity());
                 logger.fine("Response from " + get.getURI().toString() + " is: " + responseString);
                 JsonObject responseJson = JsonUtil.getJsonObject(responseString);
-                return (long) responseJson.getInt("size");
+                return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getInt("size");
             } else {
-                logger.warning("Response from " + get.getURI().toString() + " was " + response.getStatusLine().getStatusCode());
+                logger.warning("Response from " + get.getURI().toString() + " was "
+                        + response.getStatusLine().getStatusCode());
                 logger.info(EntityUtils.toString(response.getEntity()));
             }
         } catch (URISyntaxException e) {
@@ -227,24 +157,6 @@ private long getSizeFromGlobus() {
             e.printStackTrace();
         }
         return -1;
-
-        /*
-         * long size = -1; HttpHead head = new HttpHead(endpointWithBasePath + "/" +
-         * path); try { CloseableHttpResponse response =
-         * getSharedHttpClient().execute(head, localContext);
-         * 
-         * try { int code = response.getStatusLine().getStatusCode();
-         * logger.fine("Response for HEAD: " + code); switch (code) { case 200: Header[]
-         * headers = response.getHeaders(HTTP.CONTENT_LEN); logger.fine("Num headers: "
-         * + headers.length); String sizeString =
-         * response.getHeaders(HTTP.CONTENT_LEN)[0].getValue();
-         * logger.fine("Content-Length: " + sizeString); size =
-         * Long.parseLong(response.getHeaders(HTTP.CONTENT_LEN)[0].getValue());
-         * logger.fine("Found file size: " + size); break; default:
-         * logger.warning("Response from " + head.getURI().toString() + " was " + code);
-         * } } finally { EntityUtils.consume(response.getEntity()); } } catch
-         * (IOException e) { logger.warning(e.getMessage()); } return size;
-         */
     }
 
     @Override
@@ -417,7 +329,7 @@ public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
     @Override
     public boolean exists() {
         logger.fine("Exists called");
-        return (getSizeFromGlobus() != -1);
+        return (retrieveSize() != -1);
     }
 
     @Override
@@ -485,9 +397,12 @@ int getUrlExpirationMinutes() {
     }
 
     private void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
-        AccessToken accessToken = GlobusServiceBean.getClientToken(JvmSettings.GLOBUS_TOKEN.lookup(driverId));
+        // String globusToken = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
+        String globusToken = System.getProperty("dataverse.files." + this.driverId + ".globus-token");
+        AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         globusAccessToken = accessToken.getOtherTokens().get(0).getAccessToken();
-        endpointWithBasePath = JvmSettings.BASE_URI.lookup(this.driverId);
+        // endpointWithBasePath = JvmSettings.BASE_URI.lookup(this.driverId);
+        endpointWithBasePath = System.getProperty("dataverse.files." + this.driverId + ".base-uri");
         logger.info("base-uri is " + endpointWithBasePath);
         if (endpointWithBasePath == null) {
             throw new IOException("dataverse.files." + this.driverId + ".base-uri is required");
@@ -527,7 +442,7 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                                 + "/" + fullStorageLocation;
                         break;
                     default:
-                        logger.warning("Not Implemented: RemoteOverlay store with base store type: "
+                        logger.warning("Not Implemented: GlobusOverlay store with base store type: "
                                 + System.getProperty("dataverse.files." + baseDriverId + ".type"));
                         throw new IOException("Not implemented");
                     }
@@ -554,7 +469,7 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                                 + "/" + fullStorageLocation;
                         break;
                     default:
-                        logger.warning("Not Implemented: RemoteOverlay store with base store type: "
+                        logger.warning("Not Implemented: GlobusOverlay store with base store type: "
                                 + System.getProperty("dataverse.files." + baseDriverId + ".type"));
                         throw new IOException("Not implemented");
                     }
@@ -640,21 +555,21 @@ private void initHttpPool() throws NoSuchAlgorithmException, KeyManagementExcept
     @Override
     public void savePath(Path fileSystemPath) throws IOException {
         throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: savePath() not implemented in this storage driver.");
+                "GlobusOverlayAccessIO: savePath() not implemented in this storage driver.");
 
     }
 
     @Override
     public void saveInputStream(InputStream inputStream) throws IOException {
         throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: saveInputStream() not implemented in this storage driver.");
+                "GlobusOverlayAccessIO: saveInputStream() not implemented in this storage driver.");
 
     }
 
     @Override
     public void saveInputStream(InputStream inputStream, Long filesize) throws IOException {
         throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: saveInputStream(InputStream, Long) not implemented in this storage driver.");
+                "GlobusOverlayAccessIO: saveInputStream(InputStream, Long) not implemented in this storage driver.");
 
     }
 
@@ -689,24 +604,42 @@ public static void main(String[] args) {
         if (args.length > 0) {
             System.out.printf("List of arguments: {}", Arrays.toString(args));
         }
-        System.setProperty("dataverse.files.globus.base-uri", "2791b83e-b989-47c5-a7fa-ce65fd949522");
+        // System.setProperty("dataverse.files.globus.globus_client_id",
+        // "2791b83e-b989-47c5-a7fa-ce65fd949522");
+        System.setProperty("dataverse.files.globus.base-uri", "d8c42580-6528-4605-9ad8-116a61982644");
         System.out.println("Valid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
-        System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOmtsa1RZc242bU1oRXNuUFFwQy9oSzQxSi9EMDV6SjRtUDd1c0ZiN011MEk9");
-        System.setProperty("dataverse.files.globus.base-store","file");
-        System.setProperty("dataverse.files.file.type",
-                DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+        // System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOkt4ZEdndFVDUDVZZG5sRG4rRHEzaVMxTHBtTVRGNlB3RjlwWm9kRTBWNVE9");
+        System.setProperty("dataverse.files.globus.globus-token",
+                "YTVlNzFjNzItYWVkYi00Mzg4LTkzNWQtY2NhM2IyODI2MzdmOnErQXRBeWNEMVM3amFWVnB0RlFnRk5zMTc3OFdDa3lGeVZPT3k0RDFpaXM9");
+        System.setProperty("dataverse.files.globus.base-store", "file");
+        System.setProperty("dataverse.files.file.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
         System.setProperty("dataverse.files.file.directory", "/tmp/files");
         logger.info(JvmSettings.BASE_URI.lookup("globus"));
-        
-        
-        
+        logger.info(JvmSettings.GLOBUS_TOKEN.lookup("globus"));
+
         try {
-            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>("globus://1234///hdc1/image001.mrc", "globus");
-        logger.info("Size is " + gsio.getSizeFromGlobus());
-        
+            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(
+                    "globus://1234///hdc1/image001.mrc", "globus");
+            logger.info("Size is " + gsio.retrieveSize());
+
         } catch (IOException e) {
             // TODO Auto-generated catch block
             e.printStackTrace();
         }
+        try {
+            DataFile df = new DataFile();
+            Dataset ds = new Dataset();
+            ds.setAuthority("10.5072");
+            ds.setIdentifier("FK21234");
+            df.setOwner(ds);
+            df.setStorageIdentifier("globus://1234///hdc1/image001.mrc");
+            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(df, null, "globus");
+            logger.info("Size2 is " + gsio.retrieveSize());
+
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index ee2b6779cba..710d7a38fb4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -85,7 +85,7 @@ public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId)
         logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
         path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
         validatePath(path);
-        
+
         logger.fine("Base URL: " + path);
     }
 
@@ -98,18 +98,17 @@ public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOE
         validatePath(path);
         logger.fine("Base URL: " + path);
     }
-    
+
     private void validatePath(String relPath) throws IOException {
         try {
             URI absoluteURI = new URI(baseUrl + "/" + relPath);
-            if(!absoluteURI.normalize().toString().startsWith(baseUrl)) {
+            if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
                 throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s base-url");
             }
-        } catch(URISyntaxException use) {
+        } catch (URISyntaxException use) {
             throw new IOException("Could not interpret storageidentifier in remote store " + this.driverId);
         }
-     }
-
+    }
 
     @Override
     public void open(DataAccessOption... options) throws IOException {
@@ -150,7 +149,7 @@ public void open(DataAccessOption... options) throws IOException {
                     this.setSize(dataFile.getFilesize());
                 } else {
                     logger.fine("Setting size");
-                    this.setSize(getSizeFromHttpHeader());
+                    this.setSize(retrieveSize());
                 }
                 if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values")
                         && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
@@ -171,16 +170,14 @@ public void open(DataAccessOption... options) throws IOException {
             }
         } else if (dvObject instanceof Dataset) {
             throw new IOException(
-                    "Data Access: RemoteOverlay Storage driver does not support dvObject type Dataverse yet");
+                    "Data Access: " + this.getClass().getName() + " does not support dvObject type Dataverse yet");
         } else if (dvObject instanceof Dataverse) {
             throw new IOException(
-                    "Data Access: RemoteOverlay Storage driver does not support dvObject type Dataverse yet");
-        } else {
-            this.setSize(getSizeFromHttpHeader());
+                    "Data Access: " + this.getClass().getName() + " does not support dvObject type Dataverse yet");
         }
     }
 
-    private long getSizeFromHttpHeader() {
+    long retrieveSize() {
         long size = -1;
         HttpHead head = new HttpHead(baseUrl + "/" + path);
         try {
@@ -356,8 +353,9 @@ public String getStorageLocation() throws IOException {
         String fullStorageLocation = dvObject.getStorageIdentifier();
         logger.fine("storageidentifier: " + fullStorageLocation);
         int driverIndex = fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR);
-        if(driverIndex >=0) {
-          fullStorageLocation = fullStorageLocation.substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        if (driverIndex >= 0) {
+            fullStorageLocation = fullStorageLocation
+                    .substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
         }
         if (this.getDvObject() instanceof Dataset) {
             throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
@@ -379,7 +377,7 @@ public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
     @Override
     public boolean exists() {
         logger.fine("Exists called");
-        return (getSizeFromHttpHeader() != -1);
+        return (retrieveSize() != -1);
     }
 
     @Override
@@ -407,7 +405,7 @@ public boolean downloadRedirectEnabled() {
         }
         return false;
     }
-    
+
     public boolean downloadRedirectEnabled(String auxObjectTag) {
         return baseStore.downloadRedirectEnabled(auxObjectTag);
     }
@@ -422,8 +420,7 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
             if (secretKey == null) {
                 return baseUrl + "/" + path;
             } else {
-                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET",
-                        secretKey);
+                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET", secretKey);
             }
         } else {
             return baseStore.generateTemporaryDownloadUrl(auxiliaryTag, auxiliaryType, auxiliaryFileName);
@@ -464,9 +461,10 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
         if (baseStore == null) {
             String baseDriverId = getBaseStoreIdFor(driverId);
             String fullStorageLocation = null;
-            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
-            
-            if(dvObject  instanceof Dataset) {
+            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type",
+                    DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+
+            if (dvObject instanceof Dataset) {
                 baseStore = DataAccess.getStorageIO(dvObject, req, baseDriverId);
             } else {
                 if (this.getDvObject() != null) {
@@ -481,8 +479,8 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                         break;
                     case DataAccess.FILE:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files") + "/"
-                                + fullStorageLocation;
+                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
+                                + "/" + fullStorageLocation;
                         break;
                     default:
                         logger.warning("Not Implemented: RemoteOverlay store with base store type: "
@@ -492,12 +490,12 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
 
                 } else if (storageLocation != null) {
                     // <remoteDriverId>://<baseStorageIdentifier>//<baseUrlPath>
-                    //remoteDriverId:// is removed if coming through directStorageIO
+                    // remoteDriverId:// is removed if coming through directStorageIO
                     int index = storageLocation.indexOf(DataAccess.SEPARATOR);
-                    if(index > 0) {
+                    if (index > 0) {
                         storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
                     }
-                    //THe base store needs the baseStoreIdentifier and not the relative URL
+                    // THe base store needs the baseStoreIdentifier and not the relative URL
                     fullStorageLocation = storageLocation.substring(0, storageLocation.indexOf("//"));
 
                     switch (baseDriverType) {
@@ -508,8 +506,8 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                         break;
                     case DataAccess.FILE:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files") + "/"
-                                + fullStorageLocation;
+                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
+                                + "/" + fullStorageLocation;
                         break;
                     default:
                         logger.warning("Not Implemented: RemoteOverlay store with base store type: "
@@ -525,37 +523,41 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
         }
         remoteStoreName = System.getProperty("dataverse.files." + this.driverId + ".remote-store-name");
         try {
-          remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
-        } catch(MalformedURLException mfue) {
+            remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
+        } catch (MalformedURLException mfue) {
             logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
         }
     }
 
-    //Convenience method to assemble the path, starting with the DOI authority/identifier/, that is needed to create a base store via DataAccess.getDirectStorageIO - the caller has to add the store type specific prefix required.
+    // Convenience method to assemble the path, starting with the DOI
+    // authority/identifier/, that is needed to create a base store via
+    // DataAccess.getDirectStorageIO - the caller has to add the store type specific
+    // prefix required.
     private String getStoragePath() throws IOException {
         String fullStoragePath = dvObject.getStorageIdentifier();
         logger.fine("storageidentifier: " + fullStoragePath);
         int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
-        if(driverIndex >=0) {
-          fullStoragePath = fullStoragePath.substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        if (driverIndex >= 0) {
+            fullStoragePath = fullStoragePath
+                    .substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
         }
         int suffixIndex = fullStoragePath.indexOf("//");
-        if(suffixIndex >=0) {
-          fullStoragePath = fullStoragePath.substring(0, suffixIndex);
+        if (suffixIndex >= 0) {
+            fullStoragePath = fullStoragePath.substring(0, suffixIndex);
         }
         if (this.getDvObject() instanceof Dataset) {
             fullStoragePath = this.getDataset().getAuthorityForFileStorage() + "/"
                     + this.getDataset().getIdentifierForFileStorage() + "/" + fullStoragePath;
         } else if (this.getDvObject() instanceof DataFile) {
             fullStoragePath = this.getDataFile().getOwner().getAuthorityForFileStorage() + "/"
-                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath; 
-        }else if (dvObject instanceof Dataverse) {
+                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath;
+        } else if (dvObject instanceof Dataverse) {
             throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
         }
         logger.fine("fullStoragePath: " + fullStoragePath);
         return fullStoragePath;
     }
-    
+
     public CloseableHttpClient getSharedHttpClient() {
         if (httpclient == null) {
             try {
@@ -617,11 +619,11 @@ protected static boolean isValidIdentifier(String driverId, String storageId) {
         String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
         try {
             URI absoluteURI = new URI(baseUrl + "/" + urlPath);
-            if(!absoluteURI.normalize().toString().startsWith(baseUrl)) {
+            if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
                 logger.warning("storageidentifier doesn't start with " + driverId + "'s base-url: " + storageId);
                 return false;
             }
-        } catch(URISyntaxException use) {
+        } catch (URISyntaxException use) {
             logger.warning("Could not interpret storageidentifier in remote store " + driverId + " : " + storageId);
             return false;
         }

From cec0b519948d8ba480f49f915dabd5f31e5c5082 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 29 Aug 2023 12:52:48 -0400
Subject: [PATCH 325/922] add globus type

---
 .../java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
index d046fa4661d..f2eb0236df4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
@@ -48,6 +48,7 @@ public DataAccess() {
     public static final String S3 = "s3";
     static final String SWIFT = "swift";
     static final String REMOTE = "remote";
+    static final String GLOBUS = "globus";
     static final String TMP = "tmp";
     public static final String SEPARATOR = "://";
     //Default to "file" is for tests only
@@ -98,6 +99,8 @@ protected static <T extends DvObject> StorageIO<T> getStorageIO(T dvObject, Data
 			return new SwiftAccessIO<>(dvObject, req, storageDriverId);
 		case REMOTE:
 			return new RemoteOverlayAccessIO<>(dvObject, req, storageDriverId);
+	     case GLOBUS:
+	            return new GlobusOverlayAccessIO<>(dvObject, req, storageDriverId);
 		case TMP:
 			throw new IOException(
 					"DataAccess IO attempted on a temporary file that hasn't been permanently saved yet.");
@@ -369,6 +372,8 @@ public static boolean isValidDirectStorageIdentifier(String storageId) {
             return S3AccessIO.isValidIdentifier(driverId, storageId);
         case REMOTE:
             return RemoteOverlayAccessIO.isValidIdentifier(driverId, storageId);
+        case GLOBUS:
+            return GlobusOverlayAccessIO.isValidIdentifier(driverId, storageId);
         default:
             logger.warning("Request to validate for storage driver: " + driverId);
         }

From 4c28979b0b62f9244313a90ce74c8a7e22791671 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 29 Aug 2023 17:48:08 -0400
Subject: [PATCH 326/922] work in progress.

---
 .../java/edu/harvard/iq/dataverse/api/Datasets.java | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 47c249b7c8a..b1858b9982f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -470,11 +470,18 @@ public Response listVersions(@Context ContainerRequestContext crc, @PathParam("i
         // @todo:  when full versions list - including files - is requested, consider 
         // using datasetservice.findDeep() (needs testing on "monstrous" datasets 
         // with a lot of versions!)
-        return response( req ->
-             ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id), offset, limit) )
+        
+        return response( req -> {
+            Dataset dataset = findDatasetOrDie(id); 
+            if (includeFiles == null ? false : includeFiles) {
+                dataset = datasetService.findDeep(dataset.getId());
+            } 
+            //return ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id), offset, limit) )
+            return ok( execCommand( new ListVersionsCommand(req, dataset, offset, limit) )
                                 .stream()
                                 .map( d -> json(d, includeFiles == null ? false : includeFiles) )
-                                .collect(toJsonArray())), getRequestUser(crc));
+                                .collect(toJsonArray()));
+        }, getRequestUser(crc));
     }
     
     @GET

From ace6783eb51c1de0995f3a1804a97446d85d801b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 30 Aug 2023 11:33:25 +0100
Subject: [PATCH 327/922] Added: getVersionFileCounts count per access status

---
 .../DatasetVersionFilesServiceBean.java       | 30 ++++++++++++++++
 .../harvard/iq/dataverse/api/Datasets.java    |  9 +++--
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 34 ++++++++++++-------
 3 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 364ac36652c..a547a216ad5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -102,6 +102,21 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
         return result;
     }
 
+    /**
+     * Given a DatasetVersion, returns its file metadata count per DataFileAccessStatus
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return Map<DataFileAccessStatus, Long> of file metadata counts per DataFileAccessStatus
+     */
+    public Map<DataFileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) {
+        Map<DataFileAccessStatus, Long> allCounts = new HashMap<>();
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Public);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Restricted);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenPublic);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenRestricted);
+        return allCounts;
+    }
+
     /**
      * Returns a FileMetadata list of files in the specified DatasetVersion
      *
@@ -144,6 +159,21 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         return baseQuery.fetch();
     }
 
+    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<DataFileAccessStatus, Long> totalCounts, DataFileAccessStatus dataFileAccessStatus) {
+        long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus);
+        if (fileMetadataCount > 0) {
+            totalCounts.put(dataFileAccessStatus, fileMetadataCount);
+        }
+    }
+
+    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, DataFileAccessStatus accessStatus) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        return queryFactory
+                .selectFrom(fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus)))
+                .stream().count();
+    }
+
     private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
         JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 297cffaa06f..b976d60b45f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -521,7 +521,6 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
         }, getRequestUser(crc));
     }
 
-    // TODO: Finalize and refine
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files/counts")
@@ -539,8 +538,12 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc, @Path
             }
             jsonObjectBuilder.add("perContentType", fileMetadataCountsPerContentTypeJsonBuilder);
 
-            // TODO
-            jsonObjectBuilder.add("perAccessStatus", 3);
+            Map<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> fileMetadataCountsPerAccessStatus = datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion);
+            JsonObjectBuilder fileMetadataCountsPerAccessStatusJsonBuilder = Json.createObjectBuilder();
+            for (Map.Entry<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> fileMetadataCount : fileMetadataCountsPerAccessStatus.entrySet()) {
+                fileMetadataCountsPerAccessStatusJsonBuilder.add(fileMetadataCount.getKey().toString(), fileMetadataCount.getValue());
+            }
+            jsonObjectBuilder.add("perAccessStatus", fileMetadataCountsPerAccessStatusJsonBuilder);
 
             Map<String, Long> fileMetadataCountsPerCategoryName = datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion);
             JsonObjectBuilder fileMetadataCountsPerCategoryNameJsonBuilder = Json.createObjectBuilder();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 1b3811a6d66..6f103df3fe8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3436,18 +3436,18 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponseRestricted.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
 
-        // Test Access Status Embargoed Then Public
+        // Test Access Status Embargoed
         UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
         String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
 
-        // Create embargo for test file 1 (Embargo and Restricted)
-        Response createExpiredFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken);
+        // Create embargo for test file 1 (Embargoed and Restricted)
+        Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken);
 
-        createExpiredFileEmbargoResponse.then().assertThat()
+        createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        // Create embargo for test file 2 (Embargo and Public)
-        Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken);
+        // Create embargo for test file 2 (Embargoed and Public)
+        createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken);
 
         createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
@@ -3523,7 +3523,14 @@ public void getVersionFileCounts() throws IOException {
         uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
         String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
         String testCategory = "testCategory";
-        UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory));
+        Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory));
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Setting embargo for file (Embargo and Public)
+        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
+        String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+        Response createFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(dataFileId), activeEmbargoDate, apiToken);
+        createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // Getting the file counts and assert each count
         Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken);
@@ -3533,10 +3540,13 @@ public void getVersionFileCounts() throws IOException {
         JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath();
         LinkedHashMap<String, Integer> responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         LinkedHashMap<String, Integer> responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
-
-        assertEquals((Integer) responseJsonPath.get("data.total"), 4);
-        assertEquals(responseCountPerContentTypeMap.get("image/png"), 2);
-        assertEquals(responseCountPerContentTypeMap.get("text/plain"), 2);
-        assertEquals(responseCountPerCategoryNameMap.get(testCategory), 1);
+        LinkedHashMap<String, Integer> responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(4, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(2, responseCountPerContentTypeMap.get("image/png"));
+        assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
+        assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(3, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString()));
+        assertEquals(1, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString()));
     }
 }

From a87136c1ae36ab83761e6cffdffe40c660b9296f Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 30 Aug 2023 11:54:16 +0100
Subject: [PATCH 328/922] Reefactor: new JsonPrinter methods for
 getVersionFileCounts response

---
 .../harvard/iq/dataverse/api/Datasets.java    | 26 +++----------------
 .../iq/dataverse/util/json/JsonPrinter.java   | 16 ++++++++++++
 2 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b976d60b45f..d082d9c29da 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -528,30 +528,10 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc, @Path
         return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-
             jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
-
-            Map<String, Long> fileMetadataCountsPerContentType = datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion);
-            JsonObjectBuilder fileMetadataCountsPerContentTypeJsonBuilder = Json.createObjectBuilder();
-            for (Map.Entry<String, Long> fileMetadataCount : fileMetadataCountsPerContentType.entrySet()) {
-                fileMetadataCountsPerContentTypeJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue());
-            }
-            jsonObjectBuilder.add("perContentType", fileMetadataCountsPerContentTypeJsonBuilder);
-
-            Map<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> fileMetadataCountsPerAccessStatus = datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion);
-            JsonObjectBuilder fileMetadataCountsPerAccessStatusJsonBuilder = Json.createObjectBuilder();
-            for (Map.Entry<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> fileMetadataCount : fileMetadataCountsPerAccessStatus.entrySet()) {
-                fileMetadataCountsPerAccessStatusJsonBuilder.add(fileMetadataCount.getKey().toString(), fileMetadataCount.getValue());
-            }
-            jsonObjectBuilder.add("perAccessStatus", fileMetadataCountsPerAccessStatusJsonBuilder);
-
-            Map<String, Long> fileMetadataCountsPerCategoryName = datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion);
-            JsonObjectBuilder fileMetadataCountsPerCategoryNameJsonBuilder = Json.createObjectBuilder();
-            for (Map.Entry<String, Long> fileMetadataCount : fileMetadataCountsPerCategoryName.entrySet()) {
-                fileMetadataCountsPerCategoryNameJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue());
-            }
-            jsonObjectBuilder.add("perCategoryName", fileMetadataCountsPerCategoryNameJsonBuilder);
-
+            jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion)));
+            jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion)));
+            jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion)));
             return ok(jsonObjectBuilder);
         }, getRequestUser(crc));
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 36085f7ead7..fb8df057dcc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -1111,6 +1111,22 @@ public Set<Collector.Characteristics> characteristics() {
         };
     }
 
+    public static JsonObjectBuilder json(Map<String, Long> map) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        for (Map.Entry<String, Long> mapEntry : map.entrySet()) {
+            jsonObjectBuilder.add(mapEntry.getKey(), mapEntry.getValue());
+        }
+        return jsonObjectBuilder;
+    }
+
+    public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> map) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        for (Map.Entry<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> mapEntry : map.entrySet()) {
+            jsonObjectBuilder.add(mapEntry.getKey().toString(), mapEntry.getValue());
+        }
+        return jsonObjectBuilder;
+    }
+
     public static Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder> toJsonArray() {
         return new Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder>() {
 

From e1913b3c024c009ad3dbbc6e1238a07c87c6863c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 30 Aug 2023 12:19:12 +0100
Subject: [PATCH 329/922] Added: docs

---
 .../9834-files-api-extension-counts.md        |  6 ++
 doc/sphinx-guides/source/api/native-api.rst   | 75 +++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 doc/release-notes/9834-files-api-extension-counts.md

diff --git a/doc/release-notes/9834-files-api-extension-counts.md b/doc/release-notes/9834-files-api-extension-counts.md
new file mode 100644
index 00000000000..3ec15d8bd36
--- /dev/null
+++ b/doc/release-notes/9834-files-api-extension-counts.md
@@ -0,0 +1,6 @@
+Implemented the following new endpoints:
+
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name). 
+
+
+- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 96555b2ba5f..8b7d34425bc 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1022,6 +1022,32 @@ Please note that both filtering and ordering criteria values are case sensitive
 
 Keep in mind that you can combine all of the above query params depending on the results you are looking for.
 
+Get File Counts in a Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get file counts, for the given dataset and version.
+
+The returned file counts are based on different criteria:
+
+- Total (The total file count)
+- Per content type
+- Per category name
+- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic)
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+  export VERSION=1.0
+
+  curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/files/counts"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts"
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -2895,6 +2921,55 @@ Also note that dataFileTags are not versioned and changes to these will update t
 
 .. _EditingVariableMetadata:
 
+Updating File Metadata Categories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates the categories for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the category names.
+
+Although updating categories can also be done with the previous endpoint, this has been created to be more practical when it is only necessary to update categories and not other metadata fields.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "$SERVER_URL/api/files/$ID/metadata/categories"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "http://demo.dataverse.org/api/files/24/metadata/categories"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000"
+
+Note that if the specified categories do not exist, they will be created.
+
 Editing Variable Level Metadata
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 555bf05af241c555300f5c528656de3d10b3c584 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 30 Aug 2023 10:07:49 -0400
Subject: [PATCH 330/922] refactoring g store as a remotestore

---
 .../dataaccess/GlobusOverlayAccessIO.java     | 370 +++---------------
 .../dataaccess/RemoteOverlayAccessIO.java     |  46 +--
 .../iq/dataverse/settings/JvmSettings.java    |   2 +-
 3 files changed, 80 insertions(+), 338 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 6a22f8b68f3..16345cd1f9c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -58,28 +58,26 @@
 /*
  * Globus Overlay Driver
  * 
- * StorageIdentifier format: <globusDriverId>://<local
- * id>//<globusEndpointIdentifier>/<absolutePath>
+ * Remote: 
+ *   StorageIdentifier format: <globusDriverId>://<baseStorageIdentifier>//<relativePath>
+ *   Storage location: <globusendpointId/basepath>/<relPath>
+ * Internal
+ *   StorageIdentifier format: <globusDriverId>://<baseStorageIdentifier>//<relativePath>
+ *   Storage location: <globusEndpointId/basepath>/<dataset authority>/<dataset identifier>/<baseStorageIdentifier>
+ *
+ * baseUrl: globus://<globusEndpointId/basePath>
+ 
  */
 public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
-    private StorageIO<DvObject> baseStore = null;
-    private String path = null;
-    private String endpointWithBasePath = null;
-
-    private static HttpClientContext localContext = HttpClientContext.create();
-    private PoolingHttpClientConnectionManager cm = null;
-    CloseableHttpClient httpclient = null;
-    private int timeout = 1200;
-    private RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
-            .setConnectionRequestTimeout(timeout * 1000).setSocketTimeout(timeout * 1000)
-            .setCookieSpec(CookieSpecs.STANDARD).setExpectContinueEnabled(true).build();
-    private static boolean trustCerts = false;
-    private int httpConcurrency = 4;
 
     private String globusAccessToken = null;
+    /*
+     * If this is set to true, the store supports Globus transfer in and Dataverse/the globus app manage file locations, access controls, deletion, etc.
+     */
+    private boolean isDataverseManaged = false;
 
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
@@ -104,9 +102,9 @@ public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOE
 
     private void validatePath(String relPath) throws IOException {
         try {
-            URI absoluteURI = new URI(endpointWithBasePath + "/" + relPath);
-            if (!absoluteURI.normalize().toString().startsWith(endpointWithBasePath)) {
-                throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s endpoint/basePath");
+            URI absoluteURI = new URI(baseUrl + "/" + relPath);
+            if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
+                throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s base-url");
             }
         } catch (URISyntaxException use) {
             throw new IOException("Could not interpret storageidentifier in remote store " + this.driverId);
@@ -120,13 +118,24 @@ long retrieveSize() {
         URI absoluteURI = null;
         try {
             int filenameStart = path.lastIndexOf("/") + 1;
+            String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
             int pathStart = endpointWithBasePath.indexOf("/");
             logger.info("endpointWithBasePath: " + endpointWithBasePath);
-            String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "")
-                    + path.substring(0, filenameStart);
+            String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart+1) : "");
             logger.info("directoryPath: " + directoryPath);
+
+            if(isDataverseManaged) {
+                Dataset ds = ((DataFile) dvObject).getOwner();
+                directoryPath = directoryPath + "/" + ds.getAuthority() + "/" + ds.getIdentifier();
+                logger.info("directoryPath now: " + directoryPath);
+
+            }
+            if(filenameStart > 0) {
+                directoryPath = directoryPath + path.substring(0, filenameStart);
+            }
+            logger.info("directoryPath finally: " + directoryPath);
             String filename = path.substring(filenameStart);
-            String endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart - 1) : endpointWithBasePath;
+            String endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
 
             absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint
                     + "/ls?path=" + directoryPath + "&filter=name:" + filename);
@@ -138,7 +147,7 @@ long retrieveSize() {
             if (response.getStatusLine().getStatusCode() == 200) {
                 // Get reponse as string
                 String responseString = EntityUtils.toString(response.getEntity());
-                logger.fine("Response from " + get.getURI().toString() + " is: " + responseString);
+                logger.info("Response from " + get.getURI().toString() + " is: " + responseString);
                 JsonObject responseJson = JsonUtil.getJsonObject(responseString);
                 return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getInt("size");
             } else {
@@ -159,63 +168,26 @@ long retrieveSize() {
         return -1;
     }
 
-    @Override
-    public InputStream getInputStream() throws IOException {
-        if (super.getInputStream() == null) {
-            try {
-                HttpGet get = new HttpGet(generateTemporaryDownloadUrl(null, null, null));
-                CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
 
-                int code = response.getStatusLine().getStatusCode();
-                switch (code) {
-                case 200:
-                    setInputStream(response.getEntity().getContent());
-                    break;
-                default:
-                    logger.warning("Response from " + get.getURI().toString() + " was " + code);
-                    throw new IOException("Cannot retrieve: " + endpointWithBasePath + "/" + path + " code: " + code);
-                }
-            } catch (Exception e) {
-                logger.warning(e.getMessage());
-                e.printStackTrace();
-                throw new IOException("Error retrieving: " + endpointWithBasePath + "/" + path + " " + e.getMessage());
 
-            }
-            setChannel(Channels.newChannel(super.getInputStream()));
-        }
-        return super.getInputStream();
-    }
-
-    @Override
-    public Channel getChannel() throws IOException {
-        if (super.getChannel() == null) {
-            getInputStream();
-        }
-        return channel;
-    }
-
-    @Override
-    public ReadableByteChannel getReadChannel() throws IOException {
-        // Make sure StorageIO.channel variable exists
-        getChannel();
-        return super.getReadChannel();
-    }
 
     @Override
     public void delete() throws IOException {
+        
+// Fix
         // Delete is best-effort - we tell the remote server and it may or may not
         // implement this call
         if (!isDirectAccess()) {
             throw new IOException("Direct Access IO must be used to permanently delete stored file objects");
         }
         try {
-            HttpDelete del = new HttpDelete(endpointWithBasePath + "/" + path);
+            HttpDelete del = new HttpDelete(baseUrl + "/" + path);
             CloseableHttpResponse response = getSharedHttpClient().execute(del, localContext);
             try {
                 int code = response.getStatusLine().getStatusCode();
                 switch (code) {
                 case 200:
-                    logger.fine("Sent DELETE for " + endpointWithBasePath + "/" + path);
+                    logger.fine("Sent DELETE for " + baseUrl + "/" + path);
                 default:
                     logger.fine("Response from DELETE on " + del.getURI().toString() + " was " + code);
                 }
@@ -224,7 +196,7 @@ public void delete() throws IOException {
             }
         } catch (Exception e) {
             logger.warning(e.getMessage());
-            throw new IOException("Error deleting: " + endpointWithBasePath + "/" + path);
+            throw new IOException("Error deleting: " + baseUrl + "/" + path);
 
         }
 
@@ -233,146 +205,20 @@ public void delete() throws IOException {
 
     }
 
-    @Override
-    public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException {
-        return baseStore.openAuxChannel(auxItemTag, options);
-    }
-
-    @Override
-    public boolean isAuxObjectCached(String auxItemTag) throws IOException {
-        return baseStore.isAuxObjectCached(auxItemTag);
-    }
-
-    @Override
-    public long getAuxObjectSize(String auxItemTag) throws IOException {
-        return baseStore.getAuxObjectSize(auxItemTag);
-    }
-
-    @Override
-    public Path getAuxObjectAsPath(String auxItemTag) throws IOException {
-        return baseStore.getAuxObjectAsPath(auxItemTag);
-    }
-
-    @Override
-    public void backupAsAux(String auxItemTag) throws IOException {
-        baseStore.backupAsAux(auxItemTag);
-    }
-
-    @Override
-    public void revertBackupAsAux(String auxItemTag) throws IOException {
-        baseStore.revertBackupAsAux(auxItemTag);
-    }
-
-    @Override
-    // this method copies a local filesystem Path into this DataAccess Auxiliary
-    // location:
-    public void savePathAsAux(Path fileSystemPath, String auxItemTag) throws IOException {
-        baseStore.savePathAsAux(fileSystemPath, auxItemTag);
-    }
-
-    @Override
-    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Long filesize) throws IOException {
-        baseStore.saveInputStreamAsAux(inputStream, auxItemTag, filesize);
-    }
-
-    /**
-     * @param inputStream InputStream we want to save
-     * @param auxItemTag  String representing this Auxiliary type ("extension")
-     * @throws IOException if anything goes wrong.
-     */
-    @Override
-    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException {
-        baseStore.saveInputStreamAsAux(inputStream, auxItemTag);
-    }
-
-    @Override
-    public List<String> listAuxObjects() throws IOException {
-        return baseStore.listAuxObjects();
-    }
-
-    @Override
-    public void deleteAuxObject(String auxItemTag) throws IOException {
-        baseStore.deleteAuxObject(auxItemTag);
-    }
-
-    @Override
-    public void deleteAllAuxObjects() throws IOException {
-        baseStore.deleteAllAuxObjects();
-    }
-
-    @Override
-    public String getStorageLocation() throws IOException {
-        String fullStorageLocation = dvObject.getStorageIdentifier();
-        logger.fine("storageidentifier: " + fullStorageLocation);
-        int driverIndex = fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR);
-        if (driverIndex >= 0) {
-            fullStorageLocation = fullStorageLocation
-                    .substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
-        }
-        if (this.getDvObject() instanceof Dataset) {
-            throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
-        } else if (this.getDvObject() instanceof DataFile) {
-            fullStorageLocation = StorageIO.getDriverPrefix(this.driverId) + fullStorageLocation;
-        } else if (dvObject instanceof Dataverse) {
-            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
-        }
-        logger.fine("fullStorageLocation: " + fullStorageLocation);
-        return fullStorageLocation;
-    }
-
-    @Override
-    public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
-        throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: this is a remote DataAccess IO object, it has no local filesystem path associated with it.");
-    }
-
-    @Override
-    public boolean exists() {
-        logger.fine("Exists called");
-        return (retrieveSize() != -1);
-    }
 
-    @Override
-    public WritableByteChannel getWriteChannel() throws UnsupportedDataAccessOperationException {
-        throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: there are no write Channels associated with S3 objects.");
-    }
 
-    @Override
-    public OutputStream getOutputStream() throws UnsupportedDataAccessOperationException {
-        throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: there are no output Streams associated with S3 objects.");
-    }
-
-    @Override
-    public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException {
-        return baseStore.getAuxFileAsInputStream(auxItemTag);
-    }
-
-    @Override
-    public boolean downloadRedirectEnabled() {
-        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".download-redirect");
-        if ("true".equalsIgnoreCase(optionValue)) {
-            return true;
-        }
-        return false;
-    }
-
-    public boolean downloadRedirectEnabled(String auxObjectTag) {
-        return baseStore.downloadRedirectEnabled(auxObjectTag);
-    }
 
     @Override
     public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName)
             throws IOException {
-
+//Fix
         // ToDo - support remote auxiliary Files
         if (auxiliaryTag == null) {
             String secretKey = System.getProperty("dataverse.files." + this.driverId + ".secret-key");
             if (secretKey == null) {
-                return endpointWithBasePath + "/" + path;
+                return baseUrl + "/" + path;
             } else {
-                return UrlSignerUtil.signUrl(endpointWithBasePath + "/" + path, getUrlExpirationMinutes(), null, "GET",
+                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET",
                         secretKey);
             }
         } else {
@@ -380,35 +226,21 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
         }
     }
 
-    int getUrlExpirationMinutes() {
-        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".url-expiration-minutes");
-        if (optionValue != null) {
-            Integer num;
-            try {
-                num = Integer.parseInt(optionValue);
-            } catch (NumberFormatException ex) {
-                num = null;
-            }
-            if (num != null) {
-                return num;
-            }
-        }
-        return 60;
-    }
-
     private void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
         // String globusToken = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
         String globusToken = System.getProperty("dataverse.files." + this.driverId + ".globus-token");
+        isDataverseManaged = Boolean.getBoolean("dataverse.files." + this.driverId + ".managed");
+
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         globusAccessToken = accessToken.getOtherTokens().get(0).getAccessToken();
         // endpointWithBasePath = JvmSettings.BASE_URI.lookup(this.driverId);
-        endpointWithBasePath = System.getProperty("dataverse.files." + this.driverId + ".base-uri");
-        logger.info("base-uri is " + endpointWithBasePath);
-        if (endpointWithBasePath == null) {
-            throw new IOException("dataverse.files." + this.driverId + ".base-uri is required");
+        baseUrl = System.getProperty("dataverse.files." + this.driverId + ".base-url");
+        logger.info("base-url is " + baseUrl);
+        if (baseUrl == null) {
+            throw new IOException("dataverse.files." + this.driverId + ".base-url is required");
         } else {
             try {
-                new URI(endpointWithBasePath);
+                new URI(baseUrl);
             } catch (Exception e) {
                 logger.warning(
                         "Trouble interpreting base-url for store: " + this.driverId + " : " + e.getLocalizedMessage());
@@ -442,9 +274,9 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                                 + "/" + fullStorageLocation;
                         break;
                     default:
-                        logger.warning("Not Implemented: GlobusOverlay store with base store type: "
+                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
                                 + System.getProperty("dataverse.files." + baseDriverId + ".type"));
-                        throw new IOException("Not implemented");
+                        throw new IOException("Not supported");
                     }
 
                 } else if (storageLocation != null) {
@@ -469,9 +301,9 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                                 + "/" + fullStorageLocation;
                         break;
                     default:
-                        logger.warning("Not Implemented: GlobusOverlay store with base store type: "
+                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
                                 + System.getProperty("dataverse.files." + baseDriverId + ".type"));
-                        throw new IOException("Not implemented");
+                        throw new IOException("Not supported");
                     }
                 }
                 baseStore = DataAccess.getDirectStorageIO(fullStorageLocation);
@@ -488,97 +320,13 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
         }
     }
 
-    // Convenience method to assemble the path, starting with the DOI
-    // authority/identifier/, that is needed to create a base store via
-    // DataAccess.getDirectStorageIO - the caller has to add the store type specific
-    // prefix required.
-    private String getStoragePath() throws IOException {
-        String fullStoragePath = dvObject.getStorageIdentifier();
-        logger.fine("storageidentifier: " + fullStoragePath);
-        int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
-        if (driverIndex >= 0) {
-            fullStoragePath = fullStoragePath
-                    .substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
-        }
-        int suffixIndex = fullStoragePath.indexOf("//");
-        if (suffixIndex >= 0) {
-            fullStoragePath = fullStoragePath.substring(0, suffixIndex);
-        }
-        if (this.getDvObject() instanceof Dataset) {
-            fullStoragePath = this.getDataset().getAuthorityForFileStorage() + "/"
-                    + this.getDataset().getIdentifierForFileStorage() + "/" + fullStoragePath;
-        } else if (this.getDvObject() instanceof DataFile) {
-            fullStoragePath = this.getDataFile().getOwner().getAuthorityForFileStorage() + "/"
-                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath;
-        } else if (dvObject instanceof Dataverse) {
-            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
-        }
-        logger.fine("fullStoragePath: " + fullStoragePath);
-        return fullStoragePath;
-    }
-
-    public CloseableHttpClient getSharedHttpClient() {
-        if (httpclient == null) {
-            try {
-                initHttpPool();
-                httpclient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(config).build();
-
-            } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException ex) {
-                logger.warning(ex.getMessage());
-            }
-        }
-        return httpclient;
-    }
-
-    private void initHttpPool() throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException {
-        if (trustCerts) {
-            // use the TrustSelfSignedStrategy to allow Self Signed Certificates
-            SSLContext sslContext;
-            SSLConnectionSocketFactory connectionFactory;
-
-            sslContext = SSLContextBuilder.create().loadTrustMaterial(new TrustAllStrategy()).build();
-            // create an SSL Socket Factory to use the SSLContext with the trust self signed
-            // certificate strategy
-            // and allow all hosts verifier.
-            connectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
-
-            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
-                    .register("https", connectionFactory).build();
-            cm = new PoolingHttpClientConnectionManager(registry);
-        } else {
-            cm = new PoolingHttpClientConnectionManager();
-        }
-        cm.setDefaultMaxPerRoute(httpConcurrency);
-        cm.setMaxTotal(httpConcurrency > 20 ? httpConcurrency : 20);
-    }
-
-    @Override
-    public void savePath(Path fileSystemPath) throws IOException {
-        throw new UnsupportedDataAccessOperationException(
-                "GlobusOverlayAccessIO: savePath() not implemented in this storage driver.");
-
-    }
-
-    @Override
-    public void saveInputStream(InputStream inputStream) throws IOException {
-        throw new UnsupportedDataAccessOperationException(
-                "GlobusOverlayAccessIO: saveInputStream() not implemented in this storage driver.");
-
-    }
-
-    @Override
-    public void saveInputStream(InputStream inputStream, Long filesize) throws IOException {
-        throw new UnsupportedDataAccessOperationException(
-                "GlobusOverlayAccessIO: saveInputStream(InputStream, Long) not implemented in this storage driver.");
-
-    }
 
     protected static boolean isValidIdentifier(String driverId, String storageId) {
         String urlPath = storageId.substring(storageId.lastIndexOf("//") + 2);
-        String baseUri = System.getProperty("dataverse.files." + driverId + ".base-uri");
+        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
         try {
-            URI absoluteURI = new URI(baseUri + "/" + urlPath);
-            if (!absoluteURI.normalize().toString().startsWith(baseUri)) {
+            URI absoluteURI = new URI(baseUrl + "/" + urlPath);
+            if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
                 logger.warning("storageidentifier doesn't start with " + driverId + "'s base-url: " + storageId);
                 return false;
             }
@@ -590,14 +338,6 @@ protected static boolean isValidIdentifier(String driverId, String storageId) {
         return true;
     }
 
-    public static String getBaseStoreIdFor(String driverId) {
-        return System.getProperty("dataverse.files." + driverId + ".base-store");
-    }
-
-    @Override
-    public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOException {
-        return baseStore.cleanUp(filter, dryRun);
-    }
 
     public static void main(String[] args) {
         System.out.println("Running the main method");
@@ -606,7 +346,7 @@ public static void main(String[] args) {
         }
         // System.setProperty("dataverse.files.globus.globus_client_id",
         // "2791b83e-b989-47c5-a7fa-ce65fd949522");
-        System.setProperty("dataverse.files.globus.base-uri", "d8c42580-6528-4605-9ad8-116a61982644");
+        System.setProperty("dataverse.files.globus.base-url", "globus://d8c42580-6528-4605-9ad8-116a61982644");
         System.out.println("Valid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
         // System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOkt4ZEdndFVDUDVZZG5sRG4rRHEzaVMxTHBtTVRGNlB3RjlwWm9kRTBWNVE9");
         System.setProperty("dataverse.files.globus.globus-token",
@@ -614,7 +354,7 @@ public static void main(String[] args) {
         System.setProperty("dataverse.files.globus.base-store", "file");
         System.setProperty("dataverse.files.file.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
         System.setProperty("dataverse.files.file.directory", "/tmp/files");
-        logger.info(JvmSettings.BASE_URI.lookup("globus"));
+        logger.info(JvmSettings.BASE_URL.lookup("globus"));
         logger.info(JvmSettings.GLOBUS_TOKEN.lookup("globus"));
 
         try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index 710d7a38fb4..6b15bcf1dc8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -52,31 +52,32 @@
 
 /**
  * @author qqmyers
- * @param <T> what it stores
  */
 /*
  * Remote Overlay Driver
  * 
  * StorageIdentifier format:
- * <httpDriverId>://<baseStorageIdentifier>//<baseUrlPath>
+ * <remoteDriverId>://<baseStorageIdentifier>//<relativePath>
+ * 
+ * baseUrl: http(s)://<host(:port)/basePath>
  */
 public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
 
-    private StorageIO<DvObject> baseStore = null;
-    private String path = null;
-    private String baseUrl = null;
+    protected StorageIO<DvObject> baseStore = null;
+    protected String path = null;
+    protected String baseUrl = null;
 
-    private static HttpClientContext localContext = HttpClientContext.create();
-    private PoolingHttpClientConnectionManager cm = null;
+    protected static HttpClientContext localContext = HttpClientContext.create();
+    protected PoolingHttpClientConnectionManager cm = null;
     CloseableHttpClient httpclient = null;
-    private int timeout = 1200;
-    private RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
+    protected int timeout = 1200;
+    protected RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
             .setConnectionRequestTimeout(timeout * 1000).setSocketTimeout(timeout * 1000)
             .setCookieSpec(CookieSpecs.STANDARD).setExpectContinueEnabled(true).build();
-    private static boolean trustCerts = false;
-    private int httpConcurrency = 4;
+    protected static boolean trustCerts = false;
+    protected int httpConcurrency = 4;
 
     public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
@@ -86,7 +87,7 @@ public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId)
         path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
         validatePath(path);
 
-        logger.fine("Base URL: " + path);
+        logger.fine("Relative path: " + path);
     }
 
     public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOException {
@@ -96,7 +97,7 @@ public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOE
 
         path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
         validatePath(path);
-        logger.fine("Base URL: " + path);
+        logger.fine("Relative path: " + path);
     }
 
     private void validatePath(String relPath) throws IOException {
@@ -420,7 +421,8 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
             if (secretKey == null) {
                 return baseUrl + "/" + path;
             } else {
-                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET", secretKey);
+                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET",
+                        secretKey);
             }
         } else {
             return baseStore.generateTemporaryDownloadUrl(auxiliaryTag, auxiliaryType, auxiliaryFileName);
@@ -483,9 +485,9 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                                 + "/" + fullStorageLocation;
                         break;
                     default:
-                        logger.warning("Not Implemented: RemoteOverlay store with base store type: "
+                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
                                 + System.getProperty("dataverse.files." + baseDriverId + ".type"));
-                        throw new IOException("Not implemented");
+                        throw new IOException("Not supported");
                     }
 
                 } else if (storageLocation != null) {
@@ -510,9 +512,9 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
                                 + "/" + fullStorageLocation;
                         break;
                     default:
-                        logger.warning("Not Implemented: RemoteOverlay store with base store type: "
+                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
                                 + System.getProperty("dataverse.files." + baseDriverId + ".type"));
-                        throw new IOException("Not implemented");
+                        throw new IOException("Not supported");
                     }
                 }
                 baseStore = DataAccess.getDirectStorageIO(fullStorageLocation);
@@ -533,7 +535,7 @@ private void configureStores(DataAccessRequest req, String driverId, String stor
     // authority/identifier/, that is needed to create a base store via
     // DataAccess.getDirectStorageIO - the caller has to add the store type specific
     // prefix required.
-    private String getStoragePath() throws IOException {
+    protected String getStoragePath() throws IOException {
         String fullStoragePath = dvObject.getStorageIdentifier();
         logger.fine("storageidentifier: " + fullStoragePath);
         int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
@@ -596,21 +598,21 @@ private void initHttpPool() throws NoSuchAlgorithmException, KeyManagementExcept
     @Override
     public void savePath(Path fileSystemPath) throws IOException {
         throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: savePath() not implemented in this storage driver.");
+                this.getClass().getName() + ": savePath() not implemented in this storage driver.");
 
     }
 
     @Override
     public void saveInputStream(InputStream inputStream) throws IOException {
         throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: saveInputStream() not implemented in this storage driver.");
+                this.getClass().getName() + ": saveInputStream() not implemented in this storage driver.");
 
     }
 
     @Override
     public void saveInputStream(InputStream inputStream, Long filesize) throws IOException {
         throw new UnsupportedDataAccessOperationException(
-                "RemoteOverlayAccessIO: saveInputStream(InputStream, Long) not implemented in this storage driver.");
+                this.getClass().getName() + ": saveInputStream(InputStream, Long) not implemented in this storage driver.");
 
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index 451dbcc56d1..ffe08a6afb9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -50,7 +50,7 @@ public enum JvmSettings {
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
     FILES(SCOPE_FILES),
-    BASE_URI(FILES, "base-uri"),
+    BASE_URL(FILES, "base-url"),
     GLOBUS_TOKEN(FILES, "globus-token"),
     
     // SOLR INDEX SETTINGS

From 87270aadbd71e18ce52cf18d32a4c7bfaf8f7257 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 30 Aug 2023 11:43:00 -0400
Subject: [PATCH 331/922] define/add ORE format version, include Dataverse
 software info

---
 .../iq/dataverse/util/bagit/OREMap.java       | 44 +++++++++++++++++--
 .../iq/dataverse/util/bagit/OREMapHelper.java |  7 ++-
 2 files changed, 46 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
index b3995b5957e..8582184d210 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
@@ -38,14 +38,33 @@
 import jakarta.json.JsonValue;
 
 import org.apache.commons.lang3.exception.ExceptionUtils;
-
+import org.bouncycastle.math.ec.ScaleYPointMap;
+
+/**
+ * This class is used to generate a JSON-LD representation of a Dataverse object leveraging the OAI_ORE and other community vocabularies. As of v1.0.0,
+ * the format is being versioned and ANY CHANGES TO THE OUTPUT of this class must be reflected in a version increment (see DATAVERSE_ORE_FORMAT_VERSION).
+ * 
+ * The OREMap class is intended to record ALL the information needed to recreate an existing Dataverse dataset. As of v1.0.0, this is true with the 
+ * exception that auxiliary files are not referenced in the OREMap. While many types of auxiliary files will be regenerated automatically based on datafile
+ *  contents, Dataverse now allows manually uploaded auxiliary files and these cannot be reproduced solely from the dataset/datafile contents. 
+ */
 public class OREMap {
 
+    //Required Services
     static SettingsServiceBean settingsService;
     static DatasetFieldServiceBean datasetFieldService;
+    static SystemConfig systemConfig;
+    
     private static final Logger logger = Logger.getLogger(OREMap.class.getCanonicalName());
     
     public static final String NAME = "OREMap";
+    
+    //NOTE: Update this value whenever the output of this class is changed
+    private static final String DATAVERSE_ORE_FORMAT_VERSION = "Dataverse OREMap Format v1.0.0";
+    private static final String DATAVERSE_SOFTWARE_NAME = "Dataverse";
+    private static final String DATAVERSE_SOFTWARE_URL = "https://github.com/iqss/dataverse";
+    
+    
     private Map<String, String> localContext = new TreeMap<String, String>();
     private DatasetVersion version;
     private Boolean excludeEmail = null;
@@ -269,10 +288,23 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) {
             return aggBuilder.add("@context", contextBuilder.build());
         } else {
             // Now create the overall map object with it's metadata
+            
+            //Start with a reference to the Dataverse software
+            JsonObjectBuilder dvSoftwareBuilder = Json.createObjectBuilder()
+                    .add("@type", JsonLDTerm.ore("SoftwareApplication").getLabel())
+                    .add(JsonLDTerm.schemaOrg("name").getLabel(), DATAVERSE_SOFTWARE_NAME)
+                    .add(JsonLDTerm.schemaOrg("version").getLabel(), systemConfig.getVersion(true))
+                    .add(JsonLDTerm.schemaOrg("url").getLabel(), DATAVERSE_SOFTWARE_URL);
+            
+            //Now the OREMAP object itself
             JsonObjectBuilder oremapBuilder = Json.createObjectBuilder()
                     .add(JsonLDTerm.dcTerms("modified").getLabel(), LocalDate.now().toString())
                     .add(JsonLDTerm.dcTerms("creator").getLabel(), BrandingUtil.getInstallationBrandName())
                     .add("@type", JsonLDTerm.ore("ResourceMap").getLabel())
+                    //Add the version of our ORE format used
+                    .add(JsonLDTerm.schemaOrg("additionalType").getLabel(), DATAVERSE_ORE_FORMAT_VERSION)
+                    //Indicate which Dataverse version created it
+                    .add(JsonLDTerm.DVCore("generatedBy").getLabel(), dvSoftwareBuilder)
                     // Define an id for the map itself (separate from the @id of the dataset being
                     // described
                     .add("@id",
@@ -283,7 +315,11 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) {
                             aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build())
                                     .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build())
                     // and finally add the context
-                    .add("@context", contextBuilder.build());
+                    .add("@context", contextBuilder.build())
+                    ;
+            
+            
+            
             return oremapBuilder;
         }
     }
@@ -467,8 +503,10 @@ private static void addCvocValue(String val, JsonArrayBuilder vals, JsonObject c
         }
     }
 
-    public static void injectSettingsService(SettingsServiceBean settingsSvc, DatasetFieldServiceBean datasetFieldSvc) {
+    //These are used to pick up various settings/constants from the application
+    public static void injectServices(SettingsServiceBean settingsSvc, DatasetFieldServiceBean datasetFieldSvc, SystemConfig systemCfg) {
         settingsService = settingsSvc;
         datasetFieldService = datasetFieldSvc;
+        systemConfig = systemCfg;
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java
index 4d63edac268..cca1e16b4f8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMapHelper.java
@@ -2,7 +2,7 @@
 
 import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
-
+import edu.harvard.iq.dataverse.util.SystemConfig;
 import jakarta.annotation.PostConstruct;
 import jakarta.ejb.EJB;
 import jakarta.ejb.Singleton;
@@ -22,8 +22,11 @@ public class OREMapHelper {
     @EJB
     DatasetFieldServiceBean datasetFieldSvc;
     
+    @EJB
+    SystemConfig systemConfig;
+    
     @PostConstruct
     public void injectService() {
-        OREMap.injectSettingsService(settingsSvc, datasetFieldSvc);
+        OREMap.injectServices(settingsSvc, datasetFieldSvc, systemConfig);
     }
 }

From 4f4355a69e909a9c6733767edfb78a660d7e59f2 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 30 Aug 2023 12:46:35 -0400
Subject: [PATCH 332/922] add version state info

---
 .../edu/harvard/iq/dataverse/util/bagit/OREMap.java | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
index 8582184d210..fbceb98f8b1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
@@ -8,6 +8,7 @@
 import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
 import edu.harvard.iq.dataverse.DatasetFieldType;
 import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObjectContainer;
 import edu.harvard.iq.dataverse.Embargo;
@@ -133,6 +134,18 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) {
                 .add(JsonLDTerm.schemaOrg("name").getLabel(), version.getTitle())
                 .add(JsonLDTerm.schemaOrg("dateModified").getLabel(), version.getLastUpdateTime().toString());
         addIfNotNull(aggBuilder, JsonLDTerm.schemaOrg("datePublished"), dataset.getPublicationDateFormattedYYYYMMDD());
+        //Add version state info - DRAFT, RELEASED, DEACCESSIONED, ARCHIVED with extra info for DEACCESIONED
+        VersionState vs = version.getVersionState();
+        if(vs.equals(VersionState.DEACCESSIONED)) {
+            JsonObjectBuilder deaccBuilder = Json.createObjectBuilder();
+            deaccBuilder.add(JsonLDTerm.schemaOrg("name").getLabel(), vs.name());
+            deaccBuilder.add(JsonLDTerm.DVCore("reason").getLabel(), version.getVersionNote());
+            addIfNotNull(deaccBuilder, JsonLDTerm.DVCore("forwardUrl"), version.getArchiveNote());
+            aggBuilder.add(JsonLDTerm.schemaOrg("creativeWorkStatus").getLabel(), deaccBuilder);
+            
+        } else {
+            aggBuilder.add(JsonLDTerm.schemaOrg("creativeWorkStatus").getLabel(), vs.name());
+        }
 
         TermsOfUseAndAccess terms = version.getTermsOfUseAndAccess();
         if (terms.getLicense() != null) {

From e69b4a517bb4e8e89dfe867f796d0c4efe56865a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 30 Aug 2023 13:03:20 -0400
Subject: [PATCH 333/922] remove unused import

---
 src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
index fbceb98f8b1..fd219bf9d93 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
@@ -39,7 +39,6 @@
 import jakarta.json.JsonValue;
 
 import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.bouncycastle.math.ec.ScaleYPointMap;
 
 /**
  * This class is used to generate a JSON-LD representation of a Dataverse object leveraging the OAI_ORE and other community vocabularies. As of v1.0.0,

From ccd6b7dfd6ea6bc5ae8ec09b0f34819f4adeda59 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 30 Aug 2023 16:58:57 -0400
Subject: [PATCH 334/922] made the "includeFiles" option true by default,
 cleaned up the ".findDeep()" logic. #9763

---
 .../edu/harvard/iq/dataverse/Dataset.java     |  8 ++--
 .../dataverse/DatasetVersionServiceBean.java  |  1 +
 .../harvard/iq/dataverse/api/Datasets.java    | 40 ++++++-------------
 .../command/impl/ListVersionsCommand.java     | 27 ++++++++++---
 4 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 620e66c6c54..a6123a36c9d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -676,11 +676,11 @@ public Timestamp getCitationDate() {
         Timestamp citationDate = null;
         //Only calculate if this dataset doesn't use an alternate date field for publication date
         if (citationDateDatasetFieldType == null) {
-            List<DatasetVersion> versions = this.versions;
+            //List<DatasetVersion> versions = this.versions;
             // TODo - is this ever not version 1.0 (or draft if not published yet)
-            DatasetVersion oldest = versions.get(versions.size() - 1);
+            //DatasetVersion oldest = versions.get(versions.size() - 1);
             citationDate = super.getPublicationDate();
-            if (oldest.isPublished()) {
+            /*if (oldest.isPublished()) {
                 List<FileMetadata> fms = oldest.getFileMetadatas();
                 for (FileMetadata fm : fms) {
                     Embargo embargo = fm.getDataFile().getEmbargo();
@@ -691,7 +691,7 @@ public Timestamp getCitationDate() {
                         }
                     }
                 }
-            }
+            }*/
         }
         return citationDate;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index fbed7d93cdd..6c514a2405c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -194,6 +194,7 @@ public DatasetVersion findDeep(Object pk) {
             .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.datasetVersion")
             .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.releaseUser")
             .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.creator")
+            .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.dataFileTags")
             .getSingleResult();
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b1858b9982f..23de46c1324 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -467,19 +467,13 @@ public Response useDefaultCitationDate(@Context ContainerRequestContext crc, @Pa
     @AuthRequired
     @Path("{id}/versions")
     public Response listVersions(@Context ContainerRequestContext crc, @PathParam("id") String id, @QueryParam("includeFiles") Boolean includeFiles, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset) {
-        // @todo:  when full versions list - including files - is requested, consider 
-        // using datasetservice.findDeep() (needs testing on "monstrous" datasets 
-        // with a lot of versions!)
         
         return response( req -> {
             Dataset dataset = findDatasetOrDie(id); 
-            if (includeFiles == null ? false : includeFiles) {
-                dataset = datasetService.findDeep(dataset.getId());
-            } 
-            //return ok( execCommand( new ListVersionsCommand(req, findDatasetOrDie(id), offset, limit) )
-            return ok( execCommand( new ListVersionsCommand(req, dataset, offset, limit) )
+
+            return ok( execCommand( new ListVersionsCommand(req, dataset, offset, limit, (includeFiles == null ? true : includeFiles)) )
                                 .stream()
-                                .map( d -> json(d, includeFiles == null ? false : includeFiles) )
+                                .map( d -> json(d, includeFiles == null ? true : includeFiles) )
                                 .collect(toJsonArray()));
         }, getRequestUser(crc));
     }
@@ -491,8 +485,15 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
         return response( req -> {
             // @todo: consider using DatasetVersionServiceBean.findDeep() here 
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
-                    : ok(json(dsv, includeFiles == null ? false : includeFiles));
+            
+            if (dsv == null || dsv.getId() == null) {
+                return notFound("Dataset version not found");
+            }
+            
+            if (includeFiles == null ? true : includeFiles) {
+                dsv = datasetversionService.findDeep(dsv.getId());
+            }
+            return ok(json(dsv, includeFiles == null ? true : includeFiles));
         }, getRequestUser(crc));
     }
     
@@ -512,23 +513,6 @@ public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam
         }, getRequestUser(crc));
     }
     
-    //@todo: remember to delete this! (for experiments only!)
-    @GET
-    @AuthRequired
-    @Path("{id}/versions/{versionId}/files2")
-    public Response getVersionFiles2(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") Long versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return response( req -> {
-            //DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
-            try {
-                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
-            } catch (IllegalArgumentException e) {
-                return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
-            }
-            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatasByDbId(versionId, limit, offset, fileMetadatasOrderCriteria)));
-        }, getRequestUser(crc));
-    }
-    
     @GET
     @AuthRequired
     @Path("{id}/dirindex")
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
index d3675a8f206..b93833ffdf9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListVersionsCommand.java
@@ -14,6 +14,7 @@
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -28,16 +29,22 @@ public class ListVersionsCommand extends AbstractCommand<List<DatasetVersion>> {
     private final Dataset ds;
     private final Integer limit; 
     private final Integer offset;
+    private final Boolean deepLookup; 
     
     public ListVersionsCommand(DataverseRequest aRequest, Dataset aDataset) {
         this(aRequest, aDataset, null, null);
     }
-
+    
     public ListVersionsCommand(DataverseRequest aRequest, Dataset aDataset, Integer offset, Integer limit) {
+        this(aRequest, aDataset, null, null, false);
+    }
+
+    public ListVersionsCommand(DataverseRequest aRequest, Dataset aDataset, Integer offset, Integer limit, boolean deepLookup) {
         super(aRequest, aDataset);
         ds = aDataset;
         this.offset = offset; 
         this.limit = limit; 
+        this.deepLookup = deepLookup; 
     }
 
     @Override
@@ -45,14 +52,22 @@ public List<DatasetVersion> execute(CommandContext ctxt) throws CommandException
         
         boolean includeUnpublished = ctxt.permissions().request(getRequest()).on(ds).has(Permission.EditDataset);
         
-        if (offset == null && limit == null) {
-            // @todo: this fragment can be dropped, and the service-based method below 
-            // can be used for both cases.
-            // @todo: on the other hand, consider using datasetservice.findDeep()
-            // when a full list of versions is requested. 
+        if (offset == null && limit == null) { 
+            
             List<DatasetVersion> outputList = new LinkedList<>();
             for (DatasetVersion dsv : ds.getVersions()) {
                 if (dsv.isReleased() || includeUnpublished) {
+                    if (deepLookup) {
+                        // @todo: when "deep"/extended lookup is requested, and 
+                        // we call .findDeep() to look up each version again, 
+                        // there is probably a more economical way to obtain the 
+                        // numeric ids of the versions, by a direct single query,
+                        // rather than go through ds.getVersions() like we are now. 
+                        dsv = ctxt.datasetVersion().findDeep(dsv.getId());
+                        if (dsv == null) {
+                            throw new CommandExecutionException("Failed to look up full list of dataset versions", this);
+                        }
+                    }
                     outputList.add(dsv);
                 }
             }

From 2d27c0392a2da21895aa9ff49bc62515ebb5faa1 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 31 Aug 2023 17:10:45 -0400
Subject: [PATCH 335/922] intermediate changes for the adjusted citation date.
 #9763

---
 .../edu/harvard/iq/dataverse/Dataset.java     | 21 +++++++++++++++++++
 .../dataverse/DatasetVersionServiceBean.java  |  1 +
 2 files changed, 22 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index a6123a36c9d..f5a2f7cc6fb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -158,6 +158,22 @@ public void setCitationDateDatasetFieldType(DatasetFieldType citationDateDataset
         this.citationDateDatasetFieldType = citationDateDatasetFieldType;
     }    
 
+    // Per DataCite best practices, the citation date of a dataset may need 
+    // to be adjusted to reflect the latest embargo availability date of any 
+    // file within the first published version. 
+    // If any files are embargoed in the first version, we will find calculate
+    // the date and cache it here. 
+    private Timestamp embargoCitationDate;
+    
+    public Timestamp getEmbargoCitationDate() {
+        return embargoCitationDate;
+    }
+
+    public void setEmbargoCitationDate(Timestamp embargoCitationDate) {
+        this.embargoCitationDate = embargoCitationDate;
+    }
+    
+    
     
     @ManyToOne
     @JoinColumn(name="template_id",nullable = true)
@@ -680,6 +696,11 @@ public Timestamp getCitationDate() {
             // TODo - is this ever not version 1.0 (or draft if not published yet)
             //DatasetVersion oldest = versions.get(versions.size() - 1);
             citationDate = super.getPublicationDate();
+            if (embargoCitationDate != null) {
+                if (citationDate.compareTo(embargoCitationDate) < 0) {
+                    return embargoCitationDate;
+                }
+            }
             /*if (oldest.isPublished()) {
                 List<FileMetadata> fms = oldest.getFileMetadatas();
                 for (FileMetadata fm : fms) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 6c514a2405c..d1a73358166 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -1309,6 +1309,7 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
                     .setHint("eclipselink.left-join-fetch", "fm.dataFile.embargo")
                     .setHint("eclipselink.left-join-fetch", "fm.datasetVersion")
                     .setHint("eclipselink.left-join-fetch", "fm.dataFile.releaseUser")
+                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.dataFileTags")
                     .setHint("eclipselink.left-join-fetch", "fm.dataFile.creator");
         } else {
             // @todo: is there really no way to use offset-limit with left join hints?

From de71e6c3532a3db911a887d2c907a8dc2c1e3956 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 5 Sep 2023 16:17:31 -0400
Subject: [PATCH 336/922] use try with resources in JsonUtil #9315

---
 .../edu/harvard/iq/dataverse/util/json/JsonUtil.java     | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index 09d02854bab..cf8b64520de 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -11,6 +11,7 @@
 import java.util.logging.Logger;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
+import jakarta.json.JsonReader;
 import jakarta.json.JsonWriter;
 import jakarta.json.JsonWriterFactory;
 import jakarta.json.stream.JsonGenerator;
@@ -60,13 +61,17 @@ public static String prettyPrint(jakarta.json.JsonObject jsonObject) {
     
     public static jakarta.json.JsonObject getJsonObject(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
-            return Json.createReader(rdr).readObject();
+            try (JsonReader jsonReader = Json.createReader(rdr)) {
+                return jsonReader.readObject();
+            }
         }
     }
     
     public static jakarta.json.JsonArray getJsonArray(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
-            return Json.createReader(rdr).readArray();
+            try (JsonReader jsonReader = Json.createReader(rdr)) {
+                return jsonReader.readArray();
+            }
         }
     }
 }

From 2ec5d3651de13db4200f4428fc69eb2b8007602c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 6 Sep 2023 10:46:17 +0100
Subject: [PATCH 337/922] Added: getZipDownloadLimit and getEmbargoEnabled API
 info endpoints

---
 .../iq/dataverse/api/AbstractApiBean.java     |  6 ++
 .../edu/harvard/iq/dataverse/api/Info.java    | 14 +++++
 .../edu/harvard/iq/dataverse/api/InfoIT.java  | 58 +++++++++++++++++--
 3 files changed, 73 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
index 5a4c9ab9058..0a0861fa1c9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
@@ -708,6 +708,12 @@ protected Response ok( boolean value ) {
             .add("data", value).build() ).build();
     }
 
+    protected Response ok(long value) {
+        return Response.ok().entity(Json.createObjectBuilder()
+                .add("status", ApiConstants.STATUS_OK)
+                .add("data", value).build()).build();
+    }
+
     /**
      * @param data Payload to return.
      * @param mediaType Non-JSON media type.
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index 3349c34dfcc..be71a3a9fc7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -65,4 +65,18 @@ public Response getTermsOfUse(@Context ContainerRequestContext crc) {
     public Response getAllowsIncompleteMetadata() {
         return ok(JvmSettings.API_ALLOW_INCOMPLETE_METADATA.lookupOptional(Boolean.class).orElse(false));
     }
+
+    @GET
+    @Path("zipDownloadLimit")
+    public Response getZipDownloadLimit() {
+        long zipDownloadLimit = SystemConfig.getLongLimitFromStringOrDefault(settingsSvc.getValueForKey(SettingsServiceBean.Key.ZipDownloadLimit), SystemConfig.defaultZipDownloadLimit);
+        return ok(zipDownloadLimit);
+    }
+
+    @GET
+    @Path("embargoEnabled")
+    public Response getEmbargoEnabled() {
+        String setting = settingsSvc.getValueForKey(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+        return ok(setting != null && !setting.equals("0"));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java b/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java
index 142b979ef3c..1b2c513252d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java
@@ -3,13 +3,27 @@
 import static io.restassured.RestAssured.given;
 import io.restassured.response.Response;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
+import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
+import static jakarta.ws.rs.core.Response.Status.OK;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
 
 public class InfoIT {
 
+    @BeforeAll
+    public static void setUpClass() {
+        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+    }
+
+    @AfterAll
+    public static void afterClass() {
+        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+    }
+
     @Test
     public void testGetDatasetPublishPopupCustomText() {
 
@@ -21,7 +35,7 @@ public void testGetDatasetPublishPopupCustomText() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/settings/" + SettingsServiceBean.Key.DatasetPublishPopupCustomText);
         response.prettyPrint();
-        response.then().assertThat().statusCode(200)
+        response.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.message", equalTo("Hello world!"));
 
         given().urlEncodingEnabled(false)
@@ -31,7 +45,7 @@ public void testGetDatasetPublishPopupCustomText() {
         response = given().urlEncodingEnabled(false)
                 .get("/api/info/settings/" + SettingsServiceBean.Key.DatasetPublishPopupCustomText);
         response.prettyPrint();
-        response.then().assertThat().statusCode(404)
+        response.then().assertThat().statusCode(NOT_FOUND.getStatusCode())
                 .body("message", equalTo("Setting "
                         + SettingsServiceBean.Key.DatasetPublishPopupCustomText
                         + " not found"));
@@ -42,7 +56,7 @@ public void testGetVersion() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/version");
         response.prettyPrint();
-        response.then().assertThat().statusCode(200)
+        response.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.version", notNullValue());
     }
 
@@ -51,7 +65,7 @@ public void testGetServer() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/server");
         response.prettyPrint();
-        response.then().assertThat().statusCode(200)
+        response.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.message", notNullValue());
     }
     
@@ -60,7 +74,41 @@ public void getTermsOfUse() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/apiTermsOfUse");
         response.prettyPrint();
-        response.then().assertThat().statusCode(200)
+        response.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.message", notNullValue());
     }
+
+    @Test
+    public void getAllowsIncompleteMetadata() {
+        Response response = given().urlEncodingEnabled(false)
+                .get("/api/info/settings/incompleteMetadataViaApi");
+        response.prettyPrint();
+        response.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data", notNullValue());
+    }
+
+    @Test
+    public void getZipDownloadLimit() {
+        Response response = given().urlEncodingEnabled(false)
+                .get("/api/info/zipDownloadLimit");
+        response.prettyPrint();
+        response.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data", notNullValue());
+    }
+
+    @Test
+    public void getEmbargoEnabled() {
+        String testEndpoint = "/api/info/embargoEnabled";
+        // Embargo disabled
+        Response response = given().urlEncodingEnabled(false).get(testEndpoint);
+        response.prettyPrint();
+        response.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data", equalTo(false));
+        // Embargo enabled
+        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
+        response = given().urlEncodingEnabled(false).get(testEndpoint);
+        response.prettyPrint();
+        response.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data", equalTo(true));
+    }
 }

From 7dd6ca8c76202a4a38d443e77e56ccf20548054f Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 6 Sep 2023 12:25:21 +0100
Subject: [PATCH 338/922] Added: docs for new info API endpoints

---
 .../9880-info-api-zip-limit-embargo.md        |  5 +++
 doc/sphinx-guides/source/api/native-api.rst   | 37 +++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 doc/release-notes/9880-info-api-zip-limit-embargo.md

diff --git a/doc/release-notes/9880-info-api-zip-limit-embargo.md b/doc/release-notes/9880-info-api-zip-limit-embargo.md
new file mode 100644
index 00000000000..6e15d503ae9
--- /dev/null
+++ b/doc/release-notes/9880-info-api-zip-limit-embargo.md
@@ -0,0 +1,5 @@
+Implemented the following new endpoints:
+
+- getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
+
+- getEmbargoEnabled (/api/info/embargoEnabled): Know if the Dataverse instance has been configured to allow embargoes.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 4d9466703e4..44486278cc8 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3330,6 +3330,43 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/info/settings/incompleteMetadataViaApi"
 
+Get Zip File Download Limit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get the configured zip file download limit. The response contains the long value of the limit in bytes.
+
+This limit comes from the database setting :ref:`:ZipDownloadLimit` if set, or the default value if the database setting is not set, which is 104857600 (100MB).
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+
+  curl "$SERVER_URL/api/info/zipDownloadLimit"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/info/zipDownloadLimit"
+
+Show Support Of The Embargo Feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Know if the Dataverse instance has been configured to allow embargoes.
+
+The endpoint checks whether the database setting :ref:`:MaxEmbargoDurationInMonths`, which enables the embargo feature, has a value that enables the feature or not.
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+
+  curl "$SERVER_URL/api/info/embargoEnabled"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/info/embargoEnabled"
 
 .. _metadata-blocks-api:
 

From e0bac9c26113da3296dffb830dbf1384bc60cc91 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 6 Sep 2023 12:33:42 +0100
Subject: [PATCH 339/922] Fixed: missing guides reference in config.rst

---
 doc/sphinx-guides/source/installation/config.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f9fe74afc7c..9fc7142333f 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -2980,6 +2980,8 @@ This setting controls the number of files that can be uploaded through the UI at
 
 ``curl -X PUT -d 500 http://localhost:8080/api/admin/settings/:MultipleUploadFilesLimit``
 
+.. _:ZipDownloadLimit:
+
 :ZipDownloadLimit
 +++++++++++++++++
 

From f61da497ffc5f9b436710e0889db193e4b6e29ab Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 13:58:25 +0200
Subject: [PATCH 340/922] typo fix

---
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 9c9e75a7f02..12069cac650 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1835,7 +1835,7 @@ public static boolean isPackageFile(DataFile dataFile) {
     public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	String driverId = dataset.getEffectiveStorageDriverId();
     	boolean directEnabled = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".aupload-out-of-band"));
+            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
     	//Should only be requested when it is allowed, but we'll log a warning otherwise
     	if(!directEnabled) {
     		logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId());

From 884d04643a0b7cad1615f348d724eb46da7d0d1b Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 14:03:22 +0200
Subject: [PATCH 341/922] edited out-of-band option description in config doc

---
 .../source/installation/config.rst            | 44 +++++++++----------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 4a099651118..14d760e5db9 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -807,28 +807,28 @@ List of S3 Storage Options
 .. table::
     :align: left
 
-    ===========================================  ==================  ==========================================================================  =============
-    JVM Option                                   Value               Description                                                                 Default value
-    ===========================================  ==================  ==========================================================================  =============
-    dataverse.files.storage-driver-id            <id>                Enable <id> as the default storage driver.                                  ``file``
-    dataverse.files.<id>.type                    ``s3``              **Required** to mark this storage as S3 based.                              (none)
-    dataverse.files.<id>.label                   <?>                 **Required** label to be shown in the UI for this storage                   (none)
-    dataverse.files.<id>.bucket-name             <?>                 The bucket name. See above.                                                 (none)
-    dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download or proxy through Dataverse.                          ``false``
-    dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset in the S3 store.           ``false``
-    dataverse.files.<id>.upload-out-of-band      ``true``/``false``  Enable direct upload of files added to a dataset via API only.              ``false``
-    dataverse.files.<id>.ingestsizelimit         <size in bytes>     Maximum size of directupload files that should be ingested                  (none)
-    dataverse.files.<id>.url-expiration-minutes  <?>                 If direct uploads/downloads: time until links expire. Optional.             60
-    dataverse.files.<id>.min-part-size           <?>                 Multipart direct uploads will occur for files larger than this. Optional.   ``1024**3``
-    dataverse.files.<id>.custom-endpoint-url     <?>                 Use custom S3 endpoint. Needs URL either with or without protocol.          (none)
-    dataverse.files.<id>.custom-endpoint-region  <?>                 Only used when using custom endpoint. Optional.                             ``dataverse``
-    dataverse.files.<id>.profile                 <?>                 Allows the use of AWS profiles for storage spanning multiple AWS accounts.  (none)
-    dataverse.files.<id>.proxy-url               <?>                 URL of a proxy protecting the S3 store. Optional.                           (none)
-    dataverse.files.<id>.path-style-access       ``true``/``false``  Use path style buckets instead of subdomains. Optional.                     ``false``
-    dataverse.files.<id>.payload-signing         ``true``/``false``  Enable payload signing. Optional                                            ``false``
-    dataverse.files.<id>.chunked-encoding        ``true``/``false``  Disable chunked encoding. Optional                                          ``true``
-    dataverse.files.<id>.connection-pool-size    <?>                 The maximum number of open connections to the S3 server                     ``256``
-    ===========================================  ==================  ==========================================================================  =============
+    ===========================================  ==================  ===================================================================================  =============
+    JVM Option                                   Value               Description                                                                          Default value
+    ===========================================  ==================  ===================================================================================  =============
+    dataverse.files.storage-driver-id            <id>                Enable <id> as the default storage driver.                                           ``file``
+    dataverse.files.<id>.type                    ``s3``              **Required** to mark this storage as S3 based.                                       (none)
+    dataverse.files.<id>.label                   <?>                 **Required** label to be shown in the UI for this storage                            (none)
+    dataverse.files.<id>.bucket-name             <?>                 The bucket name. See above.                                                          (none)
+    dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download or proxy through Dataverse.                                   ``false``
+    dataverse.files.<id>.upload-redirect         ``true``/``false``  Enable direct upload of files added to a dataset in the S3 store.                    ``false``
+    dataverse.files.<id>.upload-out-of-band      ``true``/``false``  Allow upload of files by out-of-band methods (using some tool other than Dataverse)  ``false``
+    dataverse.files.<id>.ingestsizelimit         <size in bytes>     Maximum size of directupload files that should be ingested                           (none)
+    dataverse.files.<id>.url-expiration-minutes  <?>                 If direct uploads/downloads: time until links expire. Optional.                      60
+    dataverse.files.<id>.min-part-size           <?>                 Multipart direct uploads will occur for files larger than this. Optional.            ``1024**3``
+    dataverse.files.<id>.custom-endpoint-url     <?>                 Use custom S3 endpoint. Needs URL either with or without protocol.                   (none)
+    dataverse.files.<id>.custom-endpoint-region  <?>                 Only used when using custom endpoint. Optional.                                      ``dataverse``
+    dataverse.files.<id>.profile                 <?>                 Allows the use of AWS profiles for storage spanning multiple AWS accounts.           (none)
+    dataverse.files.<id>.proxy-url               <?>                 URL of a proxy protecting the S3 store. Optional.                                    (none)
+    dataverse.files.<id>.path-style-access       ``true``/``false``  Use path style buckets instead of subdomains. Optional.                              ``false``
+    dataverse.files.<id>.payload-signing         ``true``/``false``  Enable payload signing. Optional                                                     ``false``
+    dataverse.files.<id>.chunked-encoding        ``true``/``false``  Disable chunked encoding. Optional                                                   ``true``
+    dataverse.files.<id>.connection-pool-size    <?>                 The maximum number of open connections to the S3 server                              ``256``
+    ===========================================  ==================  ===================================================================================  =============
 
 .. table::
     :align: left

From 7bb423152eac363a0bb3b3b429cdecec8eda9736 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 14:48:44 +0200
Subject: [PATCH 342/922] upload-redirect can only be true for s3 driver

---
 .../java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index 492343d1533..00db98e894e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -606,7 +606,7 @@ public static String getDriverPrefix(String driverId) {
     }
     
     public static boolean isDirectUploadEnabled(String driverId) {
-        return Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
+        return (DataAccess.S3.equals(driverId) && Boolean.parseBoolean(System.getProperty("dataverse.files." + DataAccess.S3 + ".upload-redirect"))) ||
             Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
     }
     

From f8e16cb8578931c73afe38704343ec8c83e087f8 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Sep 2023 09:14:47 -0400
Subject: [PATCH 343/922] use try with resources in JSONLDUtil #9313

---
 .../harvard/iq/dataverse/util/json/JSONLDUtil.java  | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java
index 113a6128364..4fb3ffe6c14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java
@@ -52,6 +52,7 @@
 import edu.harvard.iq.dataverse.DatasetVersion.VersionState;
 import edu.harvard.iq.dataverse.license.License;
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
+import jakarta.json.JsonReader;
 
 public class JSONLDUtil {
 
@@ -533,13 +534,11 @@ public static JsonObject decontextualizeJsonLD(String jsonLDString) {
         try (StringReader rdr = new StringReader(jsonLDString)) {
 
             // Use JsonLd to expand/compact to localContext
-            JsonObject jsonld = Json.createReader(rdr).readObject();
-            JsonDocument doc = JsonDocument.of(jsonld);
-            JsonArray array = null;
-            try {
-                array = JsonLd.expand(doc).get();
-                jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build()))
-                        .get();
+            try (JsonReader jsonReader = Json.createReader(rdr)) {
+                JsonObject jsonld = jsonReader.readObject();
+                JsonDocument doc = JsonDocument.of(jsonld);
+                JsonArray array = JsonLd.expand(doc).get();
+                jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())).get();
                 // jsonld = array.getJsonObject(0);
                 logger.fine("Decontextualized object: " + jsonld);
                 return jsonld;

From e6b7d9fab7fde94ccff5e375e9cd5f662e670bee Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 15:33:10 +0200
Subject: [PATCH 344/922] removed confusing comments about a possible error

---
 doc/sphinx-guides/source/api/native-api.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index faa3d8724f4..8529d066fab 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1552,8 +1552,6 @@ In practice, you only need one the ``dataset_id`` or the ``persistentId``. The e
     print r.json()
     print r.status_code
 
-This API call might result in the following error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. This error can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
-
 .. _add-remote-file-api:
     
 Add a Remote File to a Dataset
@@ -2748,8 +2746,6 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/:persistentId/addFiles?persistentId=doi:10.5072/FK2/7U7YBV -F jsonData='[{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}, {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]'
 
-This API call might result in the following error: "Dataset store configuration does not allow provided storageIdentifier". One of the possible causes for this error is that out of band upload is not allowed for the storage that the provided identifier refers to. This error can be resolved by enabling the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option. See :ref:`file-storage`.
-
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 

From 5ca942f1d8bdffbc730daf0d3ab7d3c368845b0c Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 15:40:50 +0200
Subject: [PATCH 345/922] removed section that was re-added in a merge, after
 it was removed in develop branch

---
 doc/sphinx-guides/source/api/native-api.rst | 43 ---------------------
 1 file changed, 43 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 8529d066fab..4d9466703e4 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2703,49 +2703,6 @@ The fully expanded example above (without environment variables) looks like this
 Note: The ``id`` returned in the json response is the id of the file metadata version.
 
 
-.. _add-file-metadata-api:
-
-Adding File Metadata
-~~~~~~~~~~~~~~~~~~~~
-
-This API call requires a ``jsonString`` expressing the metadata of multiple files. It adds file metadata to the database table where the file has already been copied to the storage.
-
-The jsonData object includes values for:
-
-* "description" - A description of the file
-* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset
-* "storageIdentifier" - String
-* "fileName" - String
-* "mimeType" - String
-* "fixity/checksum" either:
-
-  * "md5Hash" - String with MD5 hash value, or
-  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings
-
-.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below.
-
-A curl example using an ``PERSISTENT_ID``
-
-* ``SERVER_URL`` - e.g. https://demo.dataverse.org
-* ``API_TOKEN`` - API endpoints require an API token that can be passed as the X-Dataverse-key HTTP header.  For more details, see the :doc:`auth` section.
-* ``PERSISTENT_IDENTIFIER`` - Example: ``doi:10.5072/FK2/7U7YBV``
-
-.. code-block:: bash
-
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV
-  export JSON_DATA="[{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}, \
-                      {'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53', 'fileName':'file2.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123789'}}]"
-
-  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
-
-The fully expanded example above (without environment variables) looks like this:
-
-.. code-block:: bash
-
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/:persistentId/addFiles?persistentId=doi:10.5072/FK2/7U7YBV -F jsonData='[{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}, {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]'
-
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 

From 7b1e799b4eaf3d70328b5237a41dc08622112de0 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 09:54:32 -0400
Subject: [PATCH 346/922] Additional changes needed for the optimized "embargo
 publication date" aggregate. #9763

---
 .../edu/harvard/iq/dataverse/Dataset.java     |  4 +++
 .../FinalizeDatasetPublicationCommand.java    | 33 +++++++++++++++++--
 .../V6.0.0.1__9763-embargocitationdate.sql    | 14 ++++++++
 3 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index f5a2f7cc6fb..258806dad77 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -692,15 +692,19 @@ public Timestamp getCitationDate() {
         Timestamp citationDate = null;
         //Only calculate if this dataset doesn't use an alternate date field for publication date
         if (citationDateDatasetFieldType == null) {
+            // @todo: remove this commented-out code once/if the PR passes review - L.A. 
             //List<DatasetVersion> versions = this.versions;
             // TODo - is this ever not version 1.0 (or draft if not published yet)
             //DatasetVersion oldest = versions.get(versions.size() - 1);
+            // - I believe the answer is yes, the oldest versions will always be 
+            // either 1.0 or draft - L.A. 
             citationDate = super.getPublicationDate();
             if (embargoCitationDate != null) {
                 if (citationDate.compareTo(embargoCitationDate) < 0) {
                     return embargoCitationDate;
                 }
             }
+            // @todo: remove this commented-out code once/if the PR passes review - L.A. 
             /*if (oldest.isPublished()) {
                 List<FileMetadata> fms = oldest.getFileMetadatas();
                 for (FileMetadata fm : fms) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
index f5e70209744..3da087addd9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
@@ -10,6 +10,7 @@
 import edu.harvard.iq.dataverse.DatasetVersionUser;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.Embargo;
 import edu.harvard.iq.dataverse.UserNotification;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
@@ -117,9 +118,37 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
         // is this the first publication of the dataset?
         if (theDataset.getPublicationDate() == null) {
             theDataset.setReleaseUser((AuthenticatedUser) getUser());
-        }
-        if ( theDataset.getPublicationDate() == null ) {
+        
             theDataset.setPublicationDate(new Timestamp(new Date().getTime()));
+            
+            // if there are any embargoed files in this version, we will save 
+            // the latest availability date as the "embargoCitationDate" for future 
+            // reference (if the files are not available yet, as of publishing of 
+            // the dataset, this date will be used as the "ciatation date" of the dataset, 
+            // instead of the publicatonDate, in compliance with the DataCite 
+            // best practices). 
+            // the code below replicates the logic that used to be in the method 
+            // Dataset.getCitationDate() that calculated this adjusted date in real time.
+            
+            Timestamp latestEmbargoDate = null; 
+            for (DataFile dataFile : theDataset.getFiles()) {
+                // this is the first version of the dataset that is being published. 
+                // therefore we can iterate through .getFiles() instead of obtaining
+                // the DataFiles by going through the FileMetadatas in the current version.
+                Embargo embargo = dataFile.getEmbargo();
+                if (embargo != null) {
+                    // "dataAvailable" is not nullable in the Embargo class, no need for a null check
+                    Timestamp embargoDate = Timestamp.valueOf(embargo.getDateAvailable().atStartOfDay());
+                    if (latestEmbargoDate == null || latestEmbargoDate.compareTo(embargoDate) < 0) {
+                        latestEmbargoDate = embargoDate;
+                    }
+                }
+            }
+            // the above loop could be easily replaced with a database query; 
+            // but we iterate through .getFiles() elsewhere in the command, when 
+            // updating and/or registering the files, so it should not result in 
+            // an extra performance hit. 
+            theDataset.setEmbargoCitationDate(latestEmbargoDate);
         } 
 
         //Clear any external status
diff --git a/src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql b/src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql
new file mode 100644
index 00000000000..536798015ba
--- /dev/null
+++ b/src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql
@@ -0,0 +1,14 @@
+-- An aggregated timestamp which is the latest of the availability dates of any embargoed files in the first published version, if present 
+ALTER TABLE dataset ADD COLUMN IF NOT EXISTS embargoCitationDate timestamp without time zone;
+-- ... and an update query that will populate this column for all the published datasets with embargoed files in the first released version:
+UPDATE dataset SET embargocitationdate=o.embargocitationdate
+FROM (SELECT d.id, MAX(e.dateavailable) AS embargocitationdate
+FROM embargo e, dataset d, datafile f, datasetversion v, filemetadata m
+WHERE v.dataset_id = d.id
+AND v.versionstate = 'RELEASED'
+AND v.versionnumber = 1
+AND v.minorversionnumber = 0
+AND f.embargo_id = e.id
+AND m.datasetversion_id = v.id
+AND m.datafile_id = f.id GROUP BY d.id) o WHERE o.id = dataset.id;
+-- (the query follows the logic that used to be in the method Dataset.getCitationDate() that calculated this adjusted date in real time). 

From fd30fd53e521a786b59f48df786ab4b17366aa6b Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 10:19:05 -0400
Subject: [PATCH 347/922] removing a comment (#9763)

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 23de46c1324..1d7244fd6e7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -483,7 +483,6 @@ public Response listVersions(@Context ContainerRequestContext crc, @PathParam("i
     @Path("{id}/versions/{versionId}")
     public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("includeFiles") Boolean includeFiles, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         return response( req -> {
-            // @todo: consider using DatasetVersionServiceBean.findDeep() here 
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             
             if (dsv == null || dsv.getId() == null) {

From b74affc942e287329bf2aed0e7900d89fcf8bc5e Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 10:49:05 -0400
Subject: [PATCH 348/922] a short release note (#9763)

---
 doc/release-notes/9763-versions-api-improvements.md | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 doc/release-notes/9763-versions-api-improvements.md

diff --git a/doc/release-notes/9763-versions-api-improvements.md b/doc/release-notes/9763-versions-api-improvements.md
new file mode 100644
index 00000000000..2c2374dd9b6
--- /dev/null
+++ b/doc/release-notes/9763-versions-api-improvements.md
@@ -0,0 +1,4 @@
+# Some improvements have been added to the /versions API
+
+See the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide for more information.
+

From f5fba585075d499472ed7696f1be7c973f47674d Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 17:06:23 +0200
Subject: [PATCH 349/922] simplified config documentation for the out-of-band
 option

---
 doc/sphinx-guides/source/installation/config.rst | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 14d760e5db9..8080cc4d32c 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -508,12 +508,8 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
-This option allows API users to add files with the :ref:`add-file-api` call. It also allows API users to register the metadata of a file with the :ref:`add-file-metadata-api` call for a file that was uploaded directly to the storage.
-
-The option is useful in cases in which an S3 storage is not used or made public, as required by the ``dataverse.files.\<id\>.upload-redirect`` option. An example would be building a tool for synchronizing datasets with files from a third-party repository. In such a case, the tool would upload files directly to the storage, and then use :ref:`add-file-metadata-api` to link them to a dataset.
-
-Note that if a Dataverse installation uses S3-storage while the ``dataverse.files.\<id\>.upload-redirect`` JVM option is enabled, the ``upload-out-of-band`` setting is overruled. This results in direct upload being enabled even with the ``upload-out-of-band`` option not set (or set to false).
-When the ``upload-out-of-band`` option is not set to ``true`` and it isn't being overruled by the previously mentioned combination, adding files using the API will not be possible and will return the "Dataset store configuration does not allow provided storageIdentifier" error.
+By default, Dataverse support uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`add-file-metadata-api` call used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 

From 2324fe14bdc13c291fdf606ff4187183262e5f0a Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 11:10:16 -0400
Subject: [PATCH 350/922] changed the guide to reflect the fact that the
 includeFiles flag defaults to "true". (#9763)

---
 doc/sphinx-guides/source/api/native-api.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index da3fbfffa73..1234e215f0b 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -830,7 +830,7 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/datasets/24/versions?includeFiles=true"
+  curl "https://demo.dataverse.org/api/datasets/24/versions"
 
 It returns a list of versions with their metadata, and file list:
 
@@ -883,7 +883,7 @@ It returns a list of versions with their metadata, and file list:
     ]
   }
 
-The optional ``includeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``false``. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. 
+The optional ``includeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. 
 
 The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions. 
 
@@ -899,15 +899,15 @@ Get Version of a Dataset
   export ID=24
   export VERSION=1.0
 
-  curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION"
+  curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION?includeFiles=false"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0"
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?includeFiles=false"
 
-The optional ``includeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``false``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. 
+The optional ``includeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``true``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version. 
 
 
 .. _export-dataset-metadata-api:

From 43d74194f2372e3eb3a314b53def910910df36d0 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Wed, 6 Sep 2023 17:19:22 +0200
Subject: [PATCH 351/922] added references to the out-of-band option
 configuration

---
 .../source/developers/s3-direct-upload-api.rst            | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
index 4d323455d28..4bf2bbdcc79 100644
--- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
+++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
@@ -115,7 +115,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data
 
   curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
   
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. 
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
 With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
 
 To add multiple Uploaded Files to the Dataset
@@ -146,7 +146,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data
 
   curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
 
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method.
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
 With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
 
 
@@ -176,7 +176,7 @@ Note that the API call does not validate that the file matches the hash value su
 
   curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/files/$FILE_IDENTIFIER/replace" -F "jsonData=$JSON_DATA"
   
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. 
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
 With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
 
 Replacing multiple existing files in the Dataset
@@ -274,5 +274,5 @@ The JSON object returned as a response from this API call includes a "data" that
   }
 
 
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method.
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
 With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.

From 35835e40390442cac77fd5d38731b2e50d7b6560 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 11:32:42 -0400
Subject: [PATCH 352/922] extended the release note. (#9763)

---
 doc/release-notes/9763-versions-api-improvements.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/9763-versions-api-improvements.md b/doc/release-notes/9763-versions-api-improvements.md
index 2c2374dd9b6..191afe8176f 100644
--- a/doc/release-notes/9763-versions-api-improvements.md
+++ b/doc/release-notes/9763-versions-api-improvements.md
@@ -1,4 +1,8 @@
 # Some improvements have been added to the /versions API
 
-See the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide for more information.
+- optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions;
+- a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output;
+- when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files. 
+
+This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide.
 

From 9a9d7d61e95262be66970b1fda41cdfa15def540 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 11:39:10 -0400
Subject: [PATCH 353/922] cosmetic change in the release note (#9763)

---
 doc/release-notes/9763-versions-api-improvements.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/9763-versions-api-improvements.md b/doc/release-notes/9763-versions-api-improvements.md
index 191afe8176f..8d7f6c7a20a 100644
--- a/doc/release-notes/9763-versions-api-improvements.md
+++ b/doc/release-notes/9763-versions-api-improvements.md
@@ -1,4 +1,4 @@
-# Some improvements have been added to the /versions API
+# Improvements in the /versions API
 
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions;
 - a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output;

From d465b209c7cded84ff8d08799f7f4f42fb489fb2 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 11:45:58 -0400
Subject: [PATCH 354/922] cosmetic change, comment text (#9763)

---
 src/main/java/edu/harvard/iq/dataverse/Dataset.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 258806dad77..ca5a8dd2b81 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -161,8 +161,9 @@ public void setCitationDateDatasetFieldType(DatasetFieldType citationDateDataset
     // Per DataCite best practices, the citation date of a dataset may need 
     // to be adjusted to reflect the latest embargo availability date of any 
     // file within the first published version. 
-    // If any files are embargoed in the first version, we will find calculate
-    // the date and cache it here. 
+    // If any files are embargoed in the first version, this date will be
+    // calculated and cached here upon its publication, in the 
+    // FinalizeDatasetPublicationCommand. 
     private Timestamp embargoCitationDate;
     
     public Timestamp getEmbargoCitationDate() {

From ee36dee64a128942ad4412a5f64e1a1336a3063c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 6 Sep 2023 12:53:32 -0400
Subject: [PATCH 355/922] removed a noisy logging line that got checked in by
 mistake in an earlier PR, as part of a quick fix for #9803

---
 .../edu/harvard/iq/dataverse/search/SearchIncludeFragment.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 0dfad74bedf..2ce06541afa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -205,7 +205,6 @@ public String searchRedirect(String dataverseRedirectPage, Dataverse dataverseIn
          */
         
         dataverse = dataverseIn;
-        logger.info("redirect page: "+dataverseRedirectPage);
         dataverseRedirectPage = StringUtils.isBlank(dataverseRedirectPage) ? "dataverse.xhtml" : dataverseRedirectPage;
         String optionalDataverseScope = "&alias=" + dataverse.getAlias();
 

From 3bcc5557347b20625f7264ed7d760e03e69a8414 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Sep 2023 16:12:48 -0400
Subject: [PATCH 356/922] fix command for only building configbaker image

As discussed here:

https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/iterating.20on.20configbaker
---
 doc/sphinx-guides/source/container/configbaker-image.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst
index 7218e2d8d14..d098bd46436 100644
--- a/doc/sphinx-guides/source/container/configbaker-image.rst
+++ b/doc/sphinx-guides/source/container/configbaker-image.rst
@@ -86,7 +86,7 @@ Maven modules packaging target with activated "container" profile from the proje
 
 If you specifically want to build a config baker image *only*, try
 
-``mvn -Pct package -Ddocker.filter=dev_bootstrap``
+``mvn -Pct docker:build -Ddocker.filter=dev_bootstrap``
 
 The build of config baker involves copying Solr configset files. The Solr version used is inherited from Maven,
 acting as the single source of truth. Also, the tag of the image should correspond the application image, as

From 71aa91c1c9762c08efd189e8eecf4b650db25424 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Sep 2023 16:59:55 -0400
Subject: [PATCH 357/922] increase configbaker timeout

Based on discussion here:

https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/running.20using.20compose
---
 modules/container-configbaker/scripts/bootstrap.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/container-configbaker/scripts/bootstrap.sh b/modules/container-configbaker/scripts/bootstrap.sh
index 1aa9e232953..f9f97e85a5a 100644
--- a/modules/container-configbaker/scripts/bootstrap.sh
+++ b/modules/container-configbaker/scripts/bootstrap.sh
@@ -13,7 +13,7 @@ function usage() {
   echo ""
   echo "Parameters:"
   echo "instanceUrl - Location on container network where to reach your instance. Default: 'http://dataverse:8080'"
-  echo "    timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '2m'"
+  echo "    timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '3m'"
   echo "    persona - Configure persona to execute. Calls ${BOOTSTRAP_DIR}/<persona>/init.sh. Default: 'base'"
   echo ""
   echo "Note: This script will wait for the Dataverse instance to be available before executing the bootstrapping."
@@ -24,7 +24,7 @@ function usage() {
 
 # Set some defaults as documented
 DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"}
-TIMEOUT=${TIMEOUT:-"2m"}
+TIMEOUT=${TIMEOUT:-"3m"}
 
 while getopts "u:t:h" OPTION
 do

From 75b683216b1c602cd114776bbff5e78a94ed402a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 7 Sep 2023 16:32:32 -0400
Subject: [PATCH 358/922] stub out page on API design, esp paths #9880

---
 .../source/developers/api-design.rst          | 71 +++++++++++++++++++
 doc/sphinx-guides/source/developers/index.rst |  1 +
 2 files changed, 72 insertions(+)
 create mode 100755 doc/sphinx-guides/source/developers/api-design.rst

diff --git a/doc/sphinx-guides/source/developers/api-design.rst b/doc/sphinx-guides/source/developers/api-design.rst
new file mode 100755
index 00000000000..e485236223e
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/api-design.rst
@@ -0,0 +1,71 @@
+==========
+API Design
+==========
+
+API design is a large topic. We expect this page to grow over time.
+
+.. contents:: |toctitle|
+	:local:
+
+Paths
+-----
+
+A reminder `from Wikipedia <https://en.wikipedia.org/wiki/Uniform_Resource_Identifier>`_ of what a path is:
+
+.. code-block:: bash
+
+          userinfo       host      port
+          ┌──┴───� ┌──────┴──────� ┌┴�
+  https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top
+  └─┬─┘   └─────────────┬────────────┘└───────┬───────┘ └────────────┬────────────┘ └┬┘
+  scheme          authority                  path                  query           fragment
+
+Exposing Settings
+~~~~~~~~~~~~~~~~~
+
+Since Dataverse 4, database settings have been exposed via API at http://localhost:8080/api/admin/settings
+
+(JVM options are probably available via the Payara REST API, but this is out of scope.)
+
+Settings need to be exposed outside to API clients outside of ``/api/admin`` (which is typically restricted to localhost). Here are some guidelines to follow when exposing settings.
+
+- When you are exposing a database setting as-is:
+
+  - Use ``/api/info/settings`` as the root path.
+
+  - Append the name of the setting including the colon (e.g. ``:DatasetPublishPopupCustomText``)
+
+  - Final path example: ``/api/info/settings/:DatasetPublishPopupCustomText``
+
+- If the absence of the database setting is filled in by a default value (e.g. ``:ZipDownloadLimit`` or ``:ApiTermsOfUse``):
+
+  - Use ``/api/info`` as the root path.
+
+  - Append the setting but remove the colon and downcase the first character (e.g. ``zipDownloadLimit``)
+
+  - Final path example: ``/api/info/zipDownloadLimit``
+
+- If you are exposing logic based on a database setting rather that the setting itself (e.g. a boolean for if embargo is enabled rather than the value of ``:MaxEmbargoDurationInMonths``):
+
+  - Use ``/api/info`` as the root path.
+
+  - Append a meaningful name for the setting (e.g. ``embargoEnabled``).
+
+  - Final path example: ``/api/info/embargoEnabled``
+
+- If the database setting you're exposing make more sense outside of ``/api/info`` because there's more context (e.g. ``:CustomDatasetSummaryFields``):
+
+  - Feel free to use a path outside of ``/api/info`` as the root path.
+
+  - Given additional context, append a shortened name (e.g. ``/api/datasets/summaryFieldNames``).
+
+  - Final path example: ``/api/datasets/summaryFieldNames``
+
+- If you need to expose a JVM option (MicroProfile setting) such as ``dataverse.api.allow-incomplete-metadata``:
+
+  - Use ``/api/info`` as the root path.
+
+  - Append a meaningful name for the setting (e.g. ``incompleteMetadataViaApi``).
+
+  - Final path example: ``/api/info/embargoEnabled``
+
diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst
index c77ddc13519..977aeb79697 100755
--- a/doc/sphinx-guides/source/developers/index.rst
+++ b/doc/sphinx-guides/source/developers/index.rst
@@ -19,6 +19,7 @@ Developer Guide
    sql-upgrade-scripts
    testing
    documentation
+   api-design
    security
    dependencies
    debugging

From aa60eae8826bf72309e595afe7dd96a59fb44c74 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 8 Sep 2023 17:01:37 +0100
Subject: [PATCH 359/922] Added: deleted, tabularData, and fileAccessRequest
 boolean fields to DataFile API payload

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index fb8df057dcc..9bda1e24cfb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -688,9 +688,12 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 //---------------------------------------------
                 .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue()))
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
+                .add("tabularData", df.isTabularData())
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
-                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
+                .add("deleted", df.getDeleted())
+                .add("fileAccessRequest", df.getOwner().isFileAccessRequest());
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the

From 312aedd3a2cc816e686c29e071c96a47081af29e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 8 Sep 2023 17:28:33 +0100
Subject: [PATCH 360/922] Stash: userFileAccessRequested endpoint WIP

---
 .../edu/harvard/iq/dataverse/api/Access.java  | 59 +++++++++++++------
 1 file changed, 41 insertions(+), 18 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index ccdec19456c..52d97703ff9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1678,7 +1678,47 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.rejectFailure.noRequest", args));
         }
     }
-    
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userFileAccessRequested")
+    public Response userFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        AuthenticatedUser requestAuthenticatedUser;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+            requestAuthenticatedUser = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        boolean fileAccessRequested = false;
+        List<FileAccessRequest> requests = dataFile.getFileAccessRequests();
+        for (FileAccessRequest fileAccessRequest : requests) {
+            if (fileAccessRequest.getAuthenticatedUser().getId().equals(requestAuthenticatedUser.getId())) {
+                fileAccessRequested = true;
+                break;
+            }
+        }
+        return ok(fileAccessRequested);
+    }
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userPermissions")
+    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        User requestUser = getRequestUser(crc);
+        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
+        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
+        return ok(jsonObjectBuilder);
+    }
+
     // checkAuthorization is a convenience method; it calls the boolean method
     // isAccessAuthorized(), the actual workhorse, tand throws a 403 exception if not.
     
@@ -1946,21 +1986,4 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin
         }
         return redirectUri;
     }
-
-    @GET
-    @AuthRequired
-    @Path("/datafile/{id}/userPermissions")
-    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        User requestUser = getRequestUser(crc);
-        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
-        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
-        return ok(jsonObjectBuilder);
-    }
 }

From 455cb2c950eab61bcaa25a8251cef186e8ea3837 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 8 Sep 2023 18:10:49 +0100
Subject: [PATCH 361/922] Fixed: removed deleted field from DataFile payload
 which causes nullability issues

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 9bda1e24cfb..2b04bb3f657 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -692,7 +692,6 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
                 .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
-                .add("deleted", df.getDeleted())
                 .add("fileAccessRequest", df.getOwner().isFileAccessRequest());
         /*
          * The restricted state was not included prior to #9175 so to avoid backward

From 55a81be1c6b279a253b12782a18e7b7c1db9bb9e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 8 Sep 2023 18:12:34 +0100
Subject: [PATCH 362/922] Refactor: simpler IT testGetUserPermissionsOnFile

---
 .../harvard/iq/dataverse/api/AccessIT.java    | 23 ++-----------------
 1 file changed, 2 insertions(+), 21 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index 76012882ef5..dadd4093fc2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -633,27 +633,8 @@ public void testZipUploadAndDownload() throws IOException {
 
     @Test
     public void testGetUserPermissionsOnFile() {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Upload test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Assert user permissions on file
-        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(testFileId), apiToken);
-
+        // Call with valid file id
+        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(basicFileId), apiToken);
         getUserPermissionsOnFileResponse.then().assertThat().statusCode(OK.getStatusCode());
         boolean canDownloadFile = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canDownloadFile");
         assertTrue(canDownloadFile);

From 0248e1e0be00a332430563fcd762cbfdc26ac290 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Sat, 9 Sep 2023 12:02:01 +0100
Subject: [PATCH 363/922] Added: tests and tweaks for userFileAccessRequested
 API endpoint

---
 .../edu/harvard/iq/dataverse/api/Access.java  |  2 +-
 .../harvard/iq/dataverse/api/AccessIT.java    | 32 +++++++++++++++++--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 ++++
 3 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 52d97703ff9..256af8ec65e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1682,7 +1682,7 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara
     @GET
     @AuthRequired
     @Path("/datafile/{id}/userFileAccessRequested")
-    public Response userFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+    public Response getUserFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
         DataFile dataFile;
         AuthenticatedUser requestAuthenticatedUser;
         try {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index dadd4093fc2..48b6eee38e1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -26,11 +26,9 @@
 
 import static jakarta.ws.rs.core.Response.Status.*;
 import static org.hamcrest.MatcherAssert.*;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
+import static org.junit.jupiter.api.Assertions.*;
 
 /**
  *
@@ -631,6 +629,34 @@ public void testZipUploadAndDownload() throws IOException {
         System.out.println("Zip upload-and-download round trip test: success!");
     }
 
+    @Test
+    public void testGetUserFileAccessRequested() {
+        // Create new user
+        Response createUserResponse = UtilIT.createRandomUser();
+        createUserResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String newUserApiToken = UtilIT.getApiTokenFromResponse(createUserResponse);
+
+        String dataFileId = Integer.toString(tabFile3IdRestricted);
+
+        // Call with new user and unrequested access file
+        Response getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken);
+        getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        boolean userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data");
+        assertFalse(userFileAccessRequested);
+
+        // Request file access for the new user
+        Response requestFileAccessResponse = UtilIT.requestFileAccess(dataFileId, newUserApiToken);
+        requestFileAccessResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Call with new user and requested access file
+        getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken);
+        getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data");
+        assertTrue(userFileAccessRequested);
+    }
+
     @Test
     public void testGetUserPermissionsOnFile() {
         // Call with valid file id
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 973642b1617..164f3a66ffb 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3329,6 +3329,12 @@ static Response getFileDataTables(String dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/dataTables");
     }
 
+    static Response getUserFileAccessRequested(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/access/datafile/" + dataFileId + "/userFileAccessRequested");
+    }
+
     static Response getUserPermissionsOnFile(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)

From d33e8f53ffa55c1d67711ca3ea7a833574c63d7a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 11 Sep 2023 11:13:29 +0100
Subject: [PATCH 364/922] Added: hasBeenDeleted files API endpoint. Pending IT

---
 .../java/edu/harvard/iq/dataverse/api/Files.java     | 12 ++++++++++++
 .../java/edu/harvard/iq/dataverse/api/UtilIT.java    | 12 ++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 6712b68a09b..6d60de18c70 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -107,6 +107,8 @@ public class Files extends AbstractApiBean {
     MakeDataCountLoggingServiceBean mdcLogService;
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
+    @Inject
+    DataFileServiceBean dataFileServiceBean;
 
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
@@ -870,4 +872,14 @@ public Response setFileCategories(@Context ContainerRequestContext crc, @PathPar
             }
         }, getRequestUser(crc));
     }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/hasBeenDeleted")
+    public Response getHasBeenDeleted(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            return ok(dataFileServiceBean.hasBeenDeleted(dataFile));
+        }, getRequestUser(crc));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 164f3a66ffb..d243d3c47f2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3374,4 +3374,16 @@ static Response setFileCategories(String dataFileId, String apiToken, List<Strin
                 .body(jsonString)
                 .post("/api/files/" + dataFileId + "/metadata/categories");
     }
+
+    static Response deleteFileInDataset(Integer fileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .delete("/api/files/" + fileId);
+    }
+
+    static Response getHasBeenDeleted(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/files/" + dataFileId + "/hasBeenDeleted");
+    }
 }

From 972645153d1a0401196b18e1eacdc2c41b193e49 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 11 Sep 2023 12:10:23 +0100
Subject: [PATCH 365/922] Changed: :MaxEmbargoDurationInMonths setting directly
 exposed via API info endpoint

---
 doc/sphinx-guides/source/api/native-api.rst   | 16 +++--
 .../edu/harvard/iq/dataverse/api/Info.java    | 35 ++++++-----
 .../edu/harvard/iq/dataverse/api/InfoIT.java  | 59 +++++++------------
 3 files changed, 51 insertions(+), 59 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 44486278cc8..52d9099cf63 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3318,6 +3318,8 @@ Show Support Of Incomplete Metadata Deposition
 Learn if an instance has been configured to allow deposition of incomplete datasets via the API.
 See also :ref:`create-dataset-command` and :ref:`dataverse.api.allow-incomplete-metadata`
 
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
 .. code-block:: bash
 
   export SERVER_URL=https://demo.dataverse.org
@@ -3337,6 +3339,8 @@ Get the configured zip file download limit. The response contains the long value
 
 This limit comes from the database setting :ref:`:ZipDownloadLimit` if set, or the default value if the database setting is not set, which is 104857600 (100MB).
 
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
 .. code-block:: bash
 
   export SERVER_URL=https://demo.dataverse.org
@@ -3349,24 +3353,24 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/info/zipDownloadLimit"
 
-Show Support Of The Embargo Feature
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Get Maximum Embargo Duration In Months
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Know if the Dataverse instance has been configured to allow embargoes.
+Get the maximum embargo duration in months, if available, configured through the database setting :ref:`:MaxEmbargoDurationInMonths` from the Configuration section of the Installation Guide.
 
-The endpoint checks whether the database setting :ref:`:MaxEmbargoDurationInMonths`, which enables the embargo feature, has a value that enables the feature or not.
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
 
 .. code-block:: bash
 
   export SERVER_URL=https://demo.dataverse.org
 
-  curl "$SERVER_URL/api/info/embargoEnabled"
+  curl "$SERVER_URL/api/info/settings/:MaxEmbargoDurationInMonths"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/info/embargoEnabled"
+  curl "https://demo.dataverse.org/api/info/settings/:MaxEmbargoDurationInMonths"
 
 .. _metadata-blocks-api:
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index be71a3a9fc7..ccf918f1104 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -25,14 +25,15 @@ public class Info extends AbstractApiBean {
     @GET
     @Path("settings/:DatasetPublishPopupCustomText")
     public Response getDatasetPublishPopupCustomText() {
-        String setting = settingsService.getValueForKey(SettingsServiceBean.Key.DatasetPublishPopupCustomText);
-        if (setting != null) {
-            return ok(Json.createObjectBuilder().add("message", setting));
-        } else {
-            return notFound("Setting " + SettingsServiceBean.Key.DatasetPublishPopupCustomText + " not found");
-        }
+        return getSettingByKey(SettingsServiceBean.Key.DatasetPublishPopupCustomText);
     }
-    
+
+    @GET
+    @Path("settings/:MaxEmbargoDurationInMonths")
+    public Response getMaxEmbargoDurationInMonths() {
+        return getSettingByKey(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+    }
+
     @GET
     @AuthRequired
     @Path("version")
@@ -41,25 +42,25 @@ public Response getInfo(@Context ContainerRequestContext crc) {
         String[] comps = versionStr.split("build",2);
         String version = comps[0].trim();
         JsonValue build = comps.length > 1 ? Json.createArrayBuilder().add(comps[1].trim()).build().get(0) : JsonValue.NULL;
-        
+
         return response( req -> ok( Json.createObjectBuilder().add("version", version)
                                                               .add("build", build)), getRequestUser(crc));
     }
-    
+
     @GET
     @AuthRequired
     @Path("server")
     public Response getServer(@Context ContainerRequestContext crc) {
         return response( req -> ok(JvmSettings.FQDN.lookup()), getRequestUser(crc));
     }
-    
+
     @GET
     @AuthRequired
     @Path("apiTermsOfUse")
     public Response getTermsOfUse(@Context ContainerRequestContext crc) {
         return response( req -> ok(systemConfig.getApiTermsOfUse()), getRequestUser(crc));
     }
-    
+
     @GET
     @Path("settings/incompleteMetadataViaApi")
     public Response getAllowsIncompleteMetadata() {
@@ -73,10 +74,12 @@ public Response getZipDownloadLimit() {
         return ok(zipDownloadLimit);
     }
 
-    @GET
-    @Path("embargoEnabled")
-    public Response getEmbargoEnabled() {
-        String setting = settingsSvc.getValueForKey(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
-        return ok(setting != null && !setting.equals("0"));
+    private Response getSettingByKey(SettingsServiceBean.Key key) {
+        String setting = settingsService.getValueForKey(key);
+        if (setting != null) {
+            return ok(Json.createObjectBuilder().add("message", setting));
+        } else {
+            return notFound("Setting " + key + " not found");
+        }
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java b/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java
index 1b2c513252d..3d5691dbe03 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/InfoIT.java
@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.api;
 
 import static io.restassured.RestAssured.given;
+
 import io.restassured.response.Response;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import org.junit.jupiter.api.AfterAll;
@@ -17,38 +18,23 @@ public class InfoIT {
     @BeforeAll
     public static void setUpClass() {
         UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+        UtilIT.deleteSetting(SettingsServiceBean.Key.DatasetPublishPopupCustomText);
     }
 
     @AfterAll
     public static void afterClass() {
         UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+        UtilIT.deleteSetting(SettingsServiceBean.Key.DatasetPublishPopupCustomText);
     }
 
     @Test
     public void testGetDatasetPublishPopupCustomText() {
+        testSettingEndpoint(SettingsServiceBean.Key.DatasetPublishPopupCustomText, "Hello world!");
+    }
 
-        given().urlEncodingEnabled(false)
-                .body("Hello world!")
-                .put("/api/admin/settings/"
-                        + SettingsServiceBean.Key.DatasetPublishPopupCustomText);
-
-        Response response = given().urlEncodingEnabled(false)
-                .get("/api/info/settings/" + SettingsServiceBean.Key.DatasetPublishPopupCustomText);
-        response.prettyPrint();
-        response.then().assertThat().statusCode(OK.getStatusCode())
-                .body("data.message", equalTo("Hello world!"));
-
-        given().urlEncodingEnabled(false)
-                .delete("/api/admin/settings/"
-                        + SettingsServiceBean.Key.DatasetPublishPopupCustomText);
-
-        response = given().urlEncodingEnabled(false)
-                .get("/api/info/settings/" + SettingsServiceBean.Key.DatasetPublishPopupCustomText);
-        response.prettyPrint();
-        response.then().assertThat().statusCode(NOT_FOUND.getStatusCode())
-                .body("message", equalTo("Setting "
-                        + SettingsServiceBean.Key.DatasetPublishPopupCustomText
-                        + " not found"));
+    @Test
+    public void testGetMaxEmbargoDurationInMonths() {
+        testSettingEndpoint(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
     }
 
     @Test
@@ -68,9 +54,9 @@ public void testGetServer() {
         response.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.message", notNullValue());
     }
-    
+
     @Test
-    public void getTermsOfUse() {
+    public void testGetTermsOfUse() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/apiTermsOfUse");
         response.prettyPrint();
@@ -79,7 +65,7 @@ public void getTermsOfUse() {
     }
 
     @Test
-    public void getAllowsIncompleteMetadata() {
+    public void testGetAllowsIncompleteMetadata() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/settings/incompleteMetadataViaApi");
         response.prettyPrint();
@@ -88,7 +74,7 @@ public void getAllowsIncompleteMetadata() {
     }
 
     @Test
-    public void getZipDownloadLimit() {
+    public void testGetZipDownloadLimit() {
         Response response = given().urlEncodingEnabled(false)
                 .get("/api/info/zipDownloadLimit");
         response.prettyPrint();
@@ -96,19 +82,18 @@ public void getZipDownloadLimit() {
                 .body("data", notNullValue());
     }
 
-    @Test
-    public void getEmbargoEnabled() {
-        String testEndpoint = "/api/info/embargoEnabled";
-        // Embargo disabled
-        Response response = given().urlEncodingEnabled(false).get(testEndpoint);
+    private void testSettingEndpoint(SettingsServiceBean.Key settingKey, String testSettingValue) {
+        String endpoint =  "/api/info/settings/" + settingKey;
+        // Setting not found
+        Response response = given().urlEncodingEnabled(false).get(endpoint);
         response.prettyPrint();
-        response.then().assertThat().statusCode(OK.getStatusCode())
-                .body("data", equalTo(false));
-        // Embargo enabled
-        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
-        response = given().urlEncodingEnabled(false).get(testEndpoint);
+        response.then().assertThat().statusCode(NOT_FOUND.getStatusCode())
+                .body("message", equalTo("Setting " + settingKey + " not found"));
+        // Setting exists
+        UtilIT.setSetting(settingKey, testSettingValue);
+        response = given().urlEncodingEnabled(false).get(endpoint);
         response.prettyPrint();
         response.then().assertThat().statusCode(OK.getStatusCode())
-                .body("data", equalTo(true));
+                .body("data.message", equalTo(testSettingValue));
     }
 }

From d3d305c78ac579513654eb4c8a4add6d1ff8db88 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 11 Sep 2023 12:12:31 +0100
Subject: [PATCH 366/922] Changed: updated release notes

---
 doc/release-notes/9880-info-api-zip-limit-embargo.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/9880-info-api-zip-limit-embargo.md b/doc/release-notes/9880-info-api-zip-limit-embargo.md
index 6e15d503ae9..d2afb139e72 100644
--- a/doc/release-notes/9880-info-api-zip-limit-embargo.md
+++ b/doc/release-notes/9880-info-api-zip-limit-embargo.md
@@ -2,4 +2,4 @@ Implemented the following new endpoints:
 
 - getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
 
-- getEmbargoEnabled (/api/info/embargoEnabled): Know if the Dataverse instance has been configured to allow embargoes.
+- getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.

From f754b710934dc3412614ff77cde8a4c2c2efafd2 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 11 Sep 2023 12:15:02 +0100
Subject: [PATCH 367/922] Changed: private Info.java method renamed

---
 src/main/java/edu/harvard/iq/dataverse/api/Info.java | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index ccf918f1104..0652539b595 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -25,13 +25,13 @@ public class Info extends AbstractApiBean {
     @GET
     @Path("settings/:DatasetPublishPopupCustomText")
     public Response getDatasetPublishPopupCustomText() {
-        return getSettingByKey(SettingsServiceBean.Key.DatasetPublishPopupCustomText);
+        return getSettingResponseByKey(SettingsServiceBean.Key.DatasetPublishPopupCustomText);
     }
 
     @GET
     @Path("settings/:MaxEmbargoDurationInMonths")
     public Response getMaxEmbargoDurationInMonths() {
-        return getSettingByKey(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+        return getSettingResponseByKey(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
     }
 
     @GET
@@ -74,7 +74,7 @@ public Response getZipDownloadLimit() {
         return ok(zipDownloadLimit);
     }
 
-    private Response getSettingByKey(SettingsServiceBean.Key key) {
+    private Response getSettingResponseByKey(SettingsServiceBean.Key key) {
         String setting = settingsService.getValueForKey(key);
         if (setting != null) {
             return ok(Json.createObjectBuilder().add("message", setting));

From 7929c4ffae6bb3d0d9662a7d06b2e50e7bafdcb6 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Sep 2023 10:02:28 -0400
Subject: [PATCH 368/922] remove embargo example, no longer used in #9881

---
 doc/sphinx-guides/source/developers/api-design.rst | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/api-design.rst b/doc/sphinx-guides/source/developers/api-design.rst
index e485236223e..d7a14716c4d 100755
--- a/doc/sphinx-guides/source/developers/api-design.rst
+++ b/doc/sphinx-guides/source/developers/api-design.rst
@@ -45,14 +45,6 @@ Settings need to be exposed outside to API clients outside of ``/api/admin`` (wh
 
   - Final path example: ``/api/info/zipDownloadLimit``
 
-- If you are exposing logic based on a database setting rather that the setting itself (e.g. a boolean for if embargo is enabled rather than the value of ``:MaxEmbargoDurationInMonths``):
-
-  - Use ``/api/info`` as the root path.
-
-  - Append a meaningful name for the setting (e.g. ``embargoEnabled``).
-
-  - Final path example: ``/api/info/embargoEnabled``
-
 - If the database setting you're exposing make more sense outside of ``/api/info`` because there's more context (e.g. ``:CustomDatasetSummaryFields``):
 
   - Feel free to use a path outside of ``/api/info`` as the root path.

From 6e08bad0eb3e979fd50e7cbac7cfab4e2b2d5cd5 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Sep 2023 11:23:11 -0400
Subject: [PATCH 369/922] typo #9880

---
 doc/sphinx-guides/source/developers/api-design.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/developers/api-design.rst b/doc/sphinx-guides/source/developers/api-design.rst
index d7a14716c4d..e7a7a6408bb 100755
--- a/doc/sphinx-guides/source/developers/api-design.rst
+++ b/doc/sphinx-guides/source/developers/api-design.rst
@@ -59,5 +59,5 @@ Settings need to be exposed outside to API clients outside of ``/api/admin`` (wh
 
   - Append a meaningful name for the setting (e.g. ``incompleteMetadataViaApi``).
 
-  - Final path example: ``/api/info/embargoEnabled``
+  - Final path example: ``/api/info/incompleteMetadataViaApi``
 

From 1aa3703dfbbddc00356bc1a6dceb12ea577c80de Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 12:01:28 +0100
Subject: [PATCH 370/922] Added: IT for getHasBeenDeleted Files API endpoint

---
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 50 +++++++++++++++++--
 1 file changed, 46 insertions(+), 4 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index bb6b261c387..7f1ca4c8d70 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -37,10 +37,7 @@
 
 import static org.hamcrest.CoreMatchers.*;
 import static org.hamcrest.CoreMatchers.hasItem;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class FilesIT {
 
@@ -2235,4 +2232,49 @@ public void testSetFileCategories() {
                 .body("data.categories", hasItem(testCategory2))
                 .statusCode(OK.getStatusCode());
     }
+
+    @Test
+    public void testGetHasBeenDeleted() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Publish dataverse and dataset
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert that the file has not been deleted
+        Response getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken);
+        getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
+        assertFalse(fileHasBeenDeleted);
+
+        // Delete test file
+        Response deleteFileInDatasetResponse = UtilIT.deleteFileInDataset(Integer.parseInt(dataFileId), apiToken);
+        deleteFileInDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert that the file has been deleted
+        getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken);
+        getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
+        assertTrue(fileHasBeenDeleted);
+    }
 }

From c224af6fc4c3c31fe52523c5eb67b770060f8b79 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 12:19:50 +0100
Subject: [PATCH 371/922] Added: docs for userFileAccessRequested endpoint

---
 doc/sphinx-guides/source/api/dataaccess.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index d714c90372a..0bfd29ed79d 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -404,6 +404,18 @@ A curl example using an ``id``::
 
     curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests
 
+User Has Requested Access to a File:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``/api/access/datafile/{id}/userFileAccessRequested``
+
+This method returns true or false depending on whether or not the calling user has requested access to a particular file.
+
+A curl example using an ``id``::
+
+    curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userFileAccessRequested"
+
+
 Get User Permissions on a File:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 578fdc5e4f63414b85507b6605f787630cdbdfe4 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 12:27:35 +0100
Subject: [PATCH 372/922] Added: docs for hasBeenDeleted endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 37 +++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 8b7d34425bc..bfadf1eb9a4 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2872,6 +2872,43 @@ The fully expanded example above (without environment variables) looks like this
 
 If you are interested in download counts for multiple files, see :doc:`/api/metrics`.
 
+File Has Been Deleted
+~~~~~~~~~~~~~~~~~~~~~
+
+Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl "$SERVER_URL/api/files/$ID/hasBeenDeleted"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/files/24/hasBeenDeleted"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000"
+
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 

From 85b9139f94c364cf25bdf3590c5b769543f81a0f Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 12:34:19 +0100
Subject: [PATCH 373/922] Added: release notes for #9851

---
 ...-payload-extension-new-file-access-endpoints.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md

diff --git a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
new file mode 100644
index 00000000000..f306ae2ab80
--- /dev/null
+++ b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
@@ -0,0 +1,14 @@
+Implemented the following new endpoints:
+
+- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file. 
+
+
+- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
+
+
+In addition, the DataFile API payload has been extended to include the following fields:
+
+- tabularData: Boolean field to know if the DataFile is of tabular type
+
+
+- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner) 

From aacbc643b4fe7827c14edd1ebf97c82c6e0bdc53 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 12:42:22 +0100
Subject: [PATCH 374/922] Fixed: curl call examples in files API docs

---
 doc/sphinx-guides/source/api/native-api.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index bfadf1eb9a4..90f4ad4e800 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2846,13 +2846,13 @@ A curl example using an ``ID``
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
 
-  curl "$SERVER_URL/api/files/$ID/downloadCount"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/downloadCount"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/24/downloadCount"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/downloadCount"
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -2862,13 +2862,13 @@ A curl example using a ``PERSISTENT_ID``
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
-  curl "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
 
 If you are interested in download counts for multiple files, see :doc:`/api/metrics`.
 
@@ -2885,13 +2885,13 @@ A curl example using an ``ID``
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
 
-  curl "$SERVER_URL/api/files/$ID/hasBeenDeleted"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/hasBeenDeleted"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/24/hasBeenDeleted"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/hasBeenDeleted"
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -2901,13 +2901,13 @@ A curl example using a ``PERSISTENT_ID``
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
-  curl "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET  "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000"
 
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~

From 918f1fb90a6ebc6072f91a62a54223bae3ad132a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 12 Sep 2023 09:31:10 -0400
Subject: [PATCH 375/922] Revert "#9717 grant CREATE instead of ALL per
 pdurbin"

This reverts commit f71274e7c7a4d47ab7fb973320bcfdb7e6822fbd.
---
 scripts/installer/install.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/installer/install.py b/scripts/installer/install.py
index 3aedbd8c6ad..9da64bff32e 100644
--- a/scripts/installer/install.py
+++ b/scripts/installer/install.py
@@ -413,7 +413,7 @@
 
    # 3e. set permissions:
 
-   conn_cmd = "GRANT CREATE PRIVILEGES on DATABASE "+pgDb+" to "+pgUser+";"
+   conn_cmd = "GRANT ALL PRIVILEGES on DATABASE "+pgDb+" to "+pgUser+";"
    try:
       cur.execute(conn_cmd)
    except:

From 270e0fd0a28b516f62dc29e927bbb19753f47d19 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 12 Sep 2023 10:08:33 -0400
Subject: [PATCH 376/922] temporary fix for local compile issues

---
 .../harvest/server/web/servlet/OAIServlet.java    | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java
index 9cf1629abfc..3ce88fdf204 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java
@@ -74,18 +74,13 @@ public class OAIServlet extends HttpServlet {
     
     @EJB
     SystemConfig systemConfig;
-
-    @Inject
-    @ConfigProperty(name = "dataverse.oai.server.maxidentifiers", defaultValue="100")
-    private Integer maxListIdentifiers;
     
-    @Inject
-    @ConfigProperty(name = "dataverse.oai.server.maxsets", defaultValue="100")
-    private Integer maxListSets;
+    //Todo - revert this change - added to get past some local compile issues
+    private Integer maxListIdentifiers=100;
+    
+    private Integer maxListSets=100;
     
-    @Inject
-    @ConfigProperty(name = "dataverse.oai.server.maxrecords", defaultValue="10")
-    private Integer maxListRecords;
+    private Integer maxListRecords=10;
     
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.web.servlet.OAIServlet");
     // If we are going to stick with this solution - of providing a minimalist 

From 1828855a162683d564e02507ce60fd99963b43d0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 12 Sep 2023 10:09:53 -0400
Subject: [PATCH 377/922] updates/fixes re: extending RemoteOverlay, etc.

---
 .../iq/dataverse/dataaccess/DataAccess.java   |   2 +
 .../dataaccess/GlobusOverlayAccessIO.java     | 208 +++++++-----------
 .../dataaccess/RemoteOverlayAccessIO.java     |   9 +-
 .../iq/dataverse/dataaccess/StorageIO.java    |   2 +-
 .../dataverse/globus/GlobusServiceBean.java   |  52 +++--
 5 files changed, 119 insertions(+), 154 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
index f2eb0236df4..8387f8110cf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
@@ -132,6 +132,8 @@ public static StorageIO<DvObject> getDirectStorageIO(String fullStorageLocation)
             return new SwiftAccessIO<>(storageLocation, storageDriverId);
         case REMOTE:
             return new RemoteOverlayAccessIO<>(storageLocation, storageDriverId);
+        case GLOBUS:
+            return new GlobusOverlayAccessIO<>(storageLocation, storageDriverId);
         default:
         	logger.warning("Could not find storage driver for: " + fullStorageLocation);
         	throw new IOException("getDirectStorageIO: Unsupported storage method.");
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 16345cd1f9c..b00724e2825 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -58,46 +58,63 @@
 /*
  * Globus Overlay Driver
  * 
- * Remote: 
- *   StorageIdentifier format: <globusDriverId>://<baseStorageIdentifier>//<relativePath>
- *   Storage location: <globusendpointId/basepath>/<relPath>
- * Internal
- *   StorageIdentifier format: <globusDriverId>://<baseStorageIdentifier>//<relativePath>
- *   Storage location: <globusEndpointId/basepath>/<dataset authority>/<dataset identifier>/<baseStorageIdentifier>
+ * Remote: StorageIdentifier format:
+ * <globusDriverId>://<baseStorageIdentifier>//<relativePath> Storage location:
+ * <globusendpointId/basepath>/<relPath> Internal StorageIdentifier format:
+ * <globusDriverId>://<baseStorageIdentifier> Storage location:
+ * <globusEndpointId/basepath>/<dataset authority>/<dataset
+ * identifier>/<baseStorageIdentifier>
  *
  * baseUrl: globus://<globusEndpointId/basePath>
- 
+ * 
  */
 public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
-
-    private String globusAccessToken = null;
+    String globusAccessToken = null;
     /*
-     * If this is set to true, the store supports Globus transfer in and Dataverse/the globus app manage file locations, access controls, deletion, etc.
+     * If this is set to true, the store supports Globus transfer in and
+     * Dataverse/the globus app manage file locations, access controls, deletion,
+     * etc.
      */
-    private boolean isDataverseManaged = false;
+    private boolean dataverseManaged = false;
 
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
-        this.setIsLocalFile(false);
-        configureStores(req, driverId, null);
-        logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
-        path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
-        validatePath(path);
+        if (dvObject instanceof DataFile) {
+            globusAccessToken = retrieveGlobusAccessToken();
+        }
+        dataverseManaged = isDataverseManaged(this.driverId);
 
-        logger.fine("Relative path: " + path);
+        logger.info("GAT3: " + globusAccessToken);
     }
 
     public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
-        super(null, null, driverId);
-        this.setIsLocalFile(false);
-        configureStores(null, driverId, storageLocation);
+        this.driverId = driverId;
+        this.dataverseManaged = isDataverseManaged(this.driverId);
+        if (dataverseManaged) {
+            String[] parts = DataAccess.getDriverIdAndStorageLocation(storageLocation);
+            path = parts[1];
+        } else {
+            this.setIsLocalFile(false);
+            configureStores(null, driverId, storageLocation);
+
+            path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
+            validatePath(path);
+            logger.fine("Relative path: " + path);
+        }
+//ToDo - only when needed?
+        globusAccessToken = retrieveGlobusAccessToken();
+
+    }
+
+    private String retrieveGlobusAccessToken() {
+        // String globusToken = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
+        String globusToken = System.getProperty("dataverse.files." + this.driverId + ".globus-token");
 
-        path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
-        validatePath(path);
-        logger.fine("Relative path: " + path);
+        AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
+        return accessToken.getOtherTokens().get(0).getAccessToken();
     }
 
     private void validatePath(String relPath) throws IOException {
@@ -114,6 +131,7 @@ private void validatePath(String relPath) throws IOException {
     // Call the Globus API to get the file size
     @Override
     long retrieveSize() {
+        logger.info("GAT2: " + globusAccessToken);
         // Construct Globus URL
         URI absoluteURI = null;
         try {
@@ -121,16 +139,16 @@ long retrieveSize() {
             String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
             int pathStart = endpointWithBasePath.indexOf("/");
             logger.info("endpointWithBasePath: " + endpointWithBasePath);
-            String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart+1) : "");
+            String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
             logger.info("directoryPath: " + directoryPath);
 
-            if(isDataverseManaged) {
+            if (dataverseManaged && (dvObject!=null)) {
                 Dataset ds = ((DataFile) dvObject).getOwner();
                 directoryPath = directoryPath + "/" + ds.getAuthority() + "/" + ds.getIdentifier();
                 logger.info("directoryPath now: " + directoryPath);
 
             }
-            if(filenameStart > 0) {
+            if (filenameStart > 0) {
                 directoryPath = directoryPath + path.substring(0, filenameStart);
             }
             logger.info("directoryPath finally: " + directoryPath);
@@ -168,12 +186,15 @@ long retrieveSize() {
         return -1;
     }
 
-
-
-
+    
+    @Override
+    public InputStream getInputStream() throws IOException {
+        throw new IOException("Not implemented");
+    }
+    
     @Override
     public void delete() throws IOException {
-        
+
 // Fix
         // Delete is best-effort - we tell the remote server and it may or may not
         // implement this call
@@ -205,9 +226,6 @@ public void delete() throws IOException {
 
     }
 
-
-
-
     @Override
     public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName)
             throws IOException {
@@ -218,114 +236,37 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
             if (secretKey == null) {
                 return baseUrl + "/" + path;
             } else {
-                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET",
-                        secretKey);
+                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET", secretKey);
             }
         } else {
             return baseStore.generateTemporaryDownloadUrl(auxiliaryTag, auxiliaryType, auxiliaryFileName);
         }
     }
 
-    private void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
-        // String globusToken = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
-        String globusToken = System.getProperty("dataverse.files." + this.driverId + ".globus-token");
-        isDataverseManaged = Boolean.getBoolean("dataverse.files." + this.driverId + ".managed");
+    private static boolean isDataverseManaged(String driverId) {
+        return Boolean.getBoolean("dataverse.files." + driverId + ".managed");
+    }
 
-        AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
-        globusAccessToken = accessToken.getOtherTokens().get(0).getAccessToken();
-        // endpointWithBasePath = JvmSettings.BASE_URI.lookup(this.driverId);
-        baseUrl = System.getProperty("dataverse.files." + this.driverId + ".base-url");
-        logger.info("base-url is " + baseUrl);
+    static boolean isValidIdentifier(String driverId, String storageId) {
+        String baseIdentifier = storageId.substring(storageId.lastIndexOf("//") + 2);
+        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
         if (baseUrl == null) {
-            throw new IOException("dataverse.files." + this.driverId + ".base-url is required");
-        } else {
-            try {
-                new URI(baseUrl);
-            } catch (Exception e) {
-                logger.warning(
-                        "Trouble interpreting base-url for store: " + this.driverId + " : " + e.getLocalizedMessage());
-                throw new IOException("Can't interpret base-url as a URI");
-            }
-
+            return false;
         }
-
-        if (baseStore == null) {
-            String baseDriverId = getBaseStoreIdFor(driverId);
-            String fullStorageLocation = null;
-            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type",
-                    DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
-
-            if (dvObject instanceof Dataset) {
-                baseStore = DataAccess.getStorageIO(dvObject, req, baseDriverId);
+        // Internally managed endpoints require standard name pattern (submitted via
+        // /addFile(s) api)
+        if (isDataverseManaged(driverId)) {
+            boolean hasStandardName = usesStandardNamePattern(baseIdentifier);
+            if (hasStandardName) {
+                return true;
             } else {
-                if (this.getDvObject() != null) {
-                    fullStorageLocation = getStoragePath();
-
-                    // S3 expects <id>://<bucketname>/<key>
-                    switch (baseDriverType) {
-                    case DataAccess.S3:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".bucket-name") + "/"
-                                + fullStorageLocation;
-                        break;
-                    case DataAccess.FILE:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
-                                + "/" + fullStorageLocation;
-                        break;
-                    default:
-                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
-                                + System.getProperty("dataverse.files." + baseDriverId + ".type"));
-                        throw new IOException("Not supported");
-                    }
-
-                } else if (storageLocation != null) {
-                    // <remoteDriverId>://<baseStorageIdentifier>//<baseUrlPath>
-                    // remoteDriverId:// is removed if coming through directStorageIO
-                    int index = storageLocation.indexOf(DataAccess.SEPARATOR);
-                    if (index > 0) {
-                        storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
-                    }
-                    // THe base store needs the baseStoreIdentifier and not the relative URL
-                    fullStorageLocation = storageLocation.substring(0, storageLocation.indexOf("//"));
-
-                    switch (baseDriverType) {
-                    case DataAccess.S3:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".bucket-name") + "/"
-                                + fullStorageLocation;
-                        break;
-                    case DataAccess.FILE:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
-                                + "/" + fullStorageLocation;
-                        break;
-                    default:
-                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
-                                + System.getProperty("dataverse.files." + baseDriverId + ".type"));
-                        throw new IOException("Not supported");
-                    }
-                }
-                baseStore = DataAccess.getDirectStorageIO(fullStorageLocation);
-            }
-            if (baseDriverType.contentEquals(DataAccess.S3)) {
-                ((S3AccessIO<?>) baseStore).setMainDriver(false);
+                logger.warning("Unacceptable identifier pattern in submitted identifier: " + baseIdentifier);
+                return false;
             }
         }
-        remoteStoreName = System.getProperty("dataverse.files." + this.driverId + ".remote-store-name");
+        // Remote endpoints require a valid URI within the baseUrl
         try {
-            remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
-        } catch (MalformedURLException mfue) {
-            logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
-        }
-    }
-
-
-    protected static boolean isValidIdentifier(String driverId, String storageId) {
-        String urlPath = storageId.substring(storageId.lastIndexOf("//") + 2);
-        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
-        try {
-            URI absoluteURI = new URI(baseUrl + "/" + urlPath);
+            URI absoluteURI = new URI(baseUrl + "/" + baseIdentifier);
             if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
                 logger.warning("storageidentifier doesn't start with " + driverId + "'s base-url: " + storageId);
                 return false;
@@ -338,7 +279,6 @@ protected static boolean isValidIdentifier(String driverId, String storageId) {
         return true;
     }
 
-
     public static void main(String[] args) {
         System.out.println("Running the main method");
         if (args.length > 0) {
@@ -347,15 +287,19 @@ public static void main(String[] args) {
         // System.setProperty("dataverse.files.globus.globus_client_id",
         // "2791b83e-b989-47c5-a7fa-ce65fd949522");
         System.setProperty("dataverse.files.globus.base-url", "globus://d8c42580-6528-4605-9ad8-116a61982644");
-        System.out.println("Valid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
+        System.out.println("NotValid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
+        System.out.println("ValidRemote: " + isValidIdentifier("globus", "globus://localid//of/the/hill"));
+        System.setProperty("dataverse.files.globus.managed", "true");
+
+        System.out.println("ValidLocal: " + isValidIdentifier("globus", "globus://176e28068b0-1c3f80357c42"));
         // System.setProperty("dataverse.files.globus.globus-token","Mjc5MWI4M2UtYjk4OS00N2M1LWE3ZmEtY2U2NWZkOTQ5NTIyOkt4ZEdndFVDUDVZZG5sRG4rRHEzaVMxTHBtTVRGNlB3RjlwWm9kRTBWNVE9");
         System.setProperty("dataverse.files.globus.globus-token",
                 "YTVlNzFjNzItYWVkYi00Mzg4LTkzNWQtY2NhM2IyODI2MzdmOnErQXRBeWNEMVM3amFWVnB0RlFnRk5zMTc3OFdDa3lGeVZPT3k0RDFpaXM9");
         System.setProperty("dataverse.files.globus.base-store", "file");
         System.setProperty("dataverse.files.file.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
         System.setProperty("dataverse.files.file.directory", "/tmp/files");
-        logger.info(JvmSettings.BASE_URL.lookup("globus"));
-        logger.info(JvmSettings.GLOBUS_TOKEN.lookup("globus"));
+        // logger.info(JvmSettings.BASE_URL.lookup("globus"));
+        // logger.info(JvmSettings.GLOBUS_TOKEN.lookup("globus"));
 
         try {
             GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index 6b15bcf1dc8..a9653f2ab68 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -65,6 +65,8 @@ public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
 
+    String globusAccessToken = null;
+    
     protected StorageIO<DvObject> baseStore = null;
     protected String path = null;
     protected String baseUrl = null;
@@ -79,6 +81,9 @@ public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
     protected static boolean trustCerts = false;
     protected int httpConcurrency = 4;
 
+    public RemoteOverlayAccessIO() {
+    }
+    
     public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
         this.setIsLocalFile(false);
@@ -445,7 +450,7 @@ int getUrlExpirationMinutes() {
         return 60;
     }
 
-    private void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
+    protected void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
         baseUrl = System.getProperty("dataverse.files." + this.driverId + ".base-url");
         if (baseUrl == null) {
             throw new IOException("dataverse.files." + this.driverId + ".base-url is required");
@@ -616,7 +621,7 @@ public void saveInputStream(InputStream inputStream, Long filesize) throws IOExc
 
     }
 
-    protected static boolean isValidIdentifier(String driverId, String storageId) {
+    static boolean isValidIdentifier(String driverId, String storageId) {
         String urlPath = storageId.substring(storageId.lastIndexOf("//") + 2);
         String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
         try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index bfd5c5f0d8f..333d72e09b2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -611,7 +611,7 @@ public static boolean isDirectUploadEnabled(String driverId) {
     
     //Check that storageIdentifier is consistent with store's config
     //False will prevent direct uploads
-    protected static boolean isValidIdentifier(String driverId, String storageId) {
+    static boolean isValidIdentifier(String driverId, String storageId) {
         return false;
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 5c387710844..d98e1c9b7f5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -594,11 +594,10 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
 
         globusLogger.info("Starting an globusUpload ");
 
-        String datasetIdentifier = dataset.getStorageIdentifier();
-
+        
         // ToDo - use DataAccess methods?
-        String storageType = datasetIdentifier.substring(0, datasetIdentifier.indexOf("://") + 3);
-        datasetIdentifier = datasetIdentifier.substring(datasetIdentifier.indexOf("://") + 3);
+        //String storageType = datasetIdentifier.substring(0, datasetIdentifier.indexOf("://") + 3);
+        //datasetIdentifier = datasetIdentifier.substring(datasetIdentifier.indexOf("://") + 3);
 
         Thread.sleep(5000);
 
@@ -670,18 +669,26 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
                 JsonArray filesJsonArray = jsonObject.getJsonArray("files");
 
                 if (filesJsonArray != null) {
+                    String datasetIdentifier = dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
 
                     for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) {
 
                         // storageIdentifier s3://gcs5-bucket1:1781cfeb8a7-748c270a227c from
                         // externalTool
                         String storageIdentifier = fileJsonObject.getString("storageIdentifier");
-                        String[] bits = storageIdentifier.split(":");
-                        String bucketName = bits[1].replace("/", "");
+                        String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier);
+                        String storeId = parts[0];
+                        //If this is an S3 store, we need to split out the bucket name
+                        String[] bits = parts[1].split(":");
+                        String bucketName = "";
+                        if(bits.length > 1) {
+                            bucketName = bits[0];
+                        }
                         String fileId = bits[bits.length - 1];
 
                         // fullpath s3://gcs5-bucket1/10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873
-                        String fullPath = storageType + bucketName + "/" + datasetIdentifier + "/" + fileId;
+                        //or globus:///10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873
+                        String fullPath = storeId + "://" + bucketName + "/" + datasetIdentifier + "/" + fileId;
                         String fileName = fileJsonObject.getString("fileName");
 
                         inputList.add(fileId + "IDsplit" + fullPath + "IDsplit" + fileName);
@@ -690,7 +697,8 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
                     // calculateMissingMetadataFields: checksum, mimetype
                     JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, globusLogger);
                     JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files");
-
+logger.info("Size: " + newfilesJsonArray.size());
+logger.info("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
                     JsonArrayBuilder jsonDataSecondAPI = Json.createArrayBuilder();
 
                     for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) {
@@ -699,15 +707,21 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
                         String storageIdentifier = fileJsonObject.getString("storageIdentifier");
                         String fileName = fileJsonObject.getString("fileName");
                         String directoryLabel = fileJsonObject.getString("directoryLabel");
-                        String[] bits = storageIdentifier.split(":");
+                        String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier);
+                        //If this is an S3 store, we need to split out the bucket name
+                        String[] bits = parts[1].split(":");
+                        String bucketName = "";
+                        if(bits.length > 1) {
+                            bucketName = bits[0];
+                        }
                         String fileId = bits[bits.length - 1];
-
+                        
                         List<JsonObject> newfileJsonObject = IntStream.range(0, newfilesJsonArray.size())
                                 .mapToObj(index -> ((JsonObject) newfilesJsonArray.get(index)).getJsonObject(fileId))
                                 .filter(Objects::nonNull).collect(Collectors.toList());
-
                         if (newfileJsonObject != null) {
-                            if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) {
+                            logger.info("List Size: " + newfileJsonObject.size());
+                            //if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) {
                                 JsonPatch path = Json.createPatchBuilder()
                                         .add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build();
                                 fileJsonObject = path.apply(fileJsonObject);
@@ -716,11 +730,11 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
                                 fileJsonObject = path.apply(fileJsonObject);
                                 jsonDataSecondAPI.add(fileJsonObject);
                                 countSuccess++;
-                            } else {
-                                globusLogger.info(fileName
-                                        + " will be skipped from adding to dataset by second API due to missing values ");
-                                countError++;
-                            }
+                           // } else {
+                           //     globusLogger.info(fileName
+                           //             + " will be skipped from adding to dataset by second API due to missing values ");
+                           //     countError++;
+                           // }
                         } else {
                             globusLogger.info(fileName
                                     + " will be skipped from adding to dataset by second API due to missing values ");
@@ -1045,8 +1059,8 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger)
             } catch (IOException ioex) {
                 count = 3;
                 logger.info(ioex.getMessage());
-                globusLogger.info("S3AccessIO: DataFile (fullPAth " + fullPath
-                        + ") does not appear to be an S3 object associated with driver: ");
+                globusLogger.info("DataFile (fullPAth " + fullPath
+                        + ") does not appear to be accessible withing Dataverse: ");
             } catch (Exception ex) {
                 count = count + 1;
                 ex.printStackTrace();

From d6727c0bb7409a195fbb1878a492f5c2b0b23d4d Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 12 Sep 2023 10:54:54 -0400
Subject: [PATCH 378/922] CREATE instead of ALL for public schema

---
 scripts/installer/install.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/installer/install.py b/scripts/installer/install.py
index 9da64bff32e..5a7b9f75696 100644
--- a/scripts/installer/install.py
+++ b/scripts/installer/install.py
@@ -422,7 +422,7 @@
    conn.close()
 
    if int(pg_major_version) >= 15:
-      conn_cmd = "GRANT ALL ON SCHEMA public TO "+pgUser+";"
+      conn_cmd = "GRANT CREATE ON SCHEMA public TO "+pgUser+";"
       print("PostgreSQL 15 or higher detected. Running " + conn_cmd)
       try:
          cur.execute(conn_cmd)

From 37136c039471d15888609724916e89723394879b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 17:14:11 +0100
Subject: [PATCH 379/922] Added: optional includeDeaccessioned parameter for
 getVersionFiles API endpoint

---
 .../harvard/iq/dataverse/api/Datasets.java    | 13 ++++++++----
 ...LatestAccessibleDatasetVersionCommand.java | 17 ++++++++-------
 ...tLatestPublishedDatasetVersionCommand.java | 21 ++++++++++++-------
 ...pecificPublishedDatasetVersionCommand.java | 18 +++++++++-------
 4 files changed, 42 insertions(+), 27 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index d082d9c29da..5064579ebfb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -501,10 +501,11 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @QueryParam("categoryName") String categoryName,
                                     @QueryParam("searchText") String searchText,
                                     @QueryParam("orderCriteria") String orderCriteria,
+                                    @QueryParam("includeDeaccessioned") boolean includeDeaccessioned,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
         return response(req -> {
-            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, includeDeaccessioned);
             DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
             try {
                 fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameAZ;
@@ -2709,11 +2710,15 @@ public static <T> T handleVersion(String versionId, DsVersionHandler<T> hdl)
     }
 
     private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, UriInfo uriInfo, HttpHeaders headers) throws WrappedResponse {
+        return getDatasetVersionOrDie(req, versionNumber, ds, uriInfo, headers, false);
+    }
+
+    private DatasetVersion getDatasetVersionOrDie(final DataverseRequest req, String versionNumber, final Dataset ds, UriInfo uriInfo, HttpHeaders headers, boolean includeDeaccessioned) throws WrappedResponse {
         DatasetVersion dsv = execCommand(handleVersion(versionNumber, new DsVersionHandler<Command<DatasetVersion>>() {
 
             @Override
             public Command<DatasetVersion> handleLatest() {
-                return new GetLatestAccessibleDatasetVersionCommand(req, ds);
+                return new GetLatestAccessibleDatasetVersionCommand(req, ds, includeDeaccessioned);
             }
 
             @Override
@@ -2723,12 +2728,12 @@ public Command<DatasetVersion> handleDraft() {
 
             @Override
             public Command<DatasetVersion> handleSpecific(long major, long minor) {
-                return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor);
+                return new GetSpecificPublishedDatasetVersionCommand(req, ds, major, minor, includeDeaccessioned);
             }
 
             @Override
             public Command<DatasetVersion> handleLatestPublished() {
-                return new GetLatestPublishedDatasetVersionCommand(req, ds);
+                return new GetLatestPublishedDatasetVersionCommand(req, ds, includeDeaccessioned);
             }
         }));
         if (dsv == null || dsv.getId() == null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java
index 680a5c3aaef..1454a4b1fdd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestAccessibleDatasetVersionCommand.java
@@ -17,29 +17,30 @@
 
 /**
  * Get the latest version of a dataset a user can view.
+ *
  * @author Naomi
  */
 // No permission needed to view published dvObjects
 @RequiredPermissions({})
-public class GetLatestAccessibleDatasetVersionCommand extends AbstractCommand<DatasetVersion>{
+public class GetLatestAccessibleDatasetVersionCommand extends AbstractCommand<DatasetVersion> {
     private final Dataset ds;
+    private final boolean includeDeaccessioned;
 
     public GetLatestAccessibleDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset) {
+        this(aRequest, anAffectedDataset, false);
+    }
+
+    public GetLatestAccessibleDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset, boolean includeDeaccessioned) {
         super(aRequest, anAffectedDataset);
         ds = anAffectedDataset;
+        this.includeDeaccessioned = includeDeaccessioned;
     }
 
     @Override
     public DatasetVersion execute(CommandContext ctxt) throws CommandException {
-
         if (ds.getLatestVersion().isDraft() && ctxt.permissions().requestOn(getRequest(), ds).has(Permission.ViewUnpublishedDataset)) {
             return ctxt.engine().submit(new GetDraftDatasetVersionCommand(getRequest(), ds));
         }
-
-        return ctxt.engine().submit(new GetLatestPublishedDatasetVersionCommand(getRequest(), ds));
-
+        return ctxt.engine().submit(new GetLatestPublishedDatasetVersionCommand(getRequest(), ds, includeDeaccessioned));
     }
-    
-    
-    
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java
index 18adff2e55c..9765d0945d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java
@@ -9,26 +9,31 @@
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 
 /**
- *
  * @author Naomi
  */
 // No permission needed to view published dvObjects
 @RequiredPermissions({})
-public class GetLatestPublishedDatasetVersionCommand extends AbstractCommand<DatasetVersion>{
+public class GetLatestPublishedDatasetVersionCommand extends AbstractCommand<DatasetVersion> {
     private final Dataset ds;
-    
+    private boolean includeDeaccessioned;
+
     public GetLatestPublishedDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset) {
+        this(aRequest, anAffectedDataset, false);
+    }
+
+    public GetLatestPublishedDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset, boolean includeDeaccessioned) {
         super(aRequest, anAffectedDataset);
         ds = anAffectedDataset;
+        this.includeDeaccessioned = includeDeaccessioned;
     }
 
     @Override
     public DatasetVersion execute(CommandContext ctxt) throws CommandException {
-        for (DatasetVersion dsv: ds.getVersions()) {
-            if (dsv.isReleased()) {
+        for (DatasetVersion dsv : ds.getVersions()) {
+            if (dsv.isReleased() || (includeDeaccessioned && dsv.isDeaccessioned())) {
                 return dsv;
-                }
             }
-        return null;
         }
-    }
\ No newline at end of file
+        return null;
+    }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java
index 3efb38e4a91..879a694ef57 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java
@@ -15,27 +15,32 @@
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 
 /**
- *
  * @author Naomi
  */
 // No permission needed to view published dvObjects
 @RequiredPermissions({})
-public class GetSpecificPublishedDatasetVersionCommand extends AbstractCommand<DatasetVersion>{
+public class GetSpecificPublishedDatasetVersionCommand extends AbstractCommand<DatasetVersion> {
     private final Dataset ds;
     private final long majorVersion;
     private final long minorVersion;
-    
+    private boolean includeDeaccessioned;
+
     public GetSpecificPublishedDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset, long majorVersionNum, long minorVersionNum) {
+        this(aRequest, anAffectedDataset, majorVersionNum, minorVersionNum, false);
+    }
+
+    public GetSpecificPublishedDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset, long majorVersionNum, long minorVersionNum, boolean includeDeaccessioned) {
         super(aRequest, anAffectedDataset);
         ds = anAffectedDataset;
         majorVersion = majorVersionNum;
         minorVersion = minorVersionNum;
+        this.includeDeaccessioned = includeDeaccessioned;
     }
 
     @Override
     public DatasetVersion execute(CommandContext ctxt) throws CommandException {
-        for (DatasetVersion dsv: ds.getVersions()) {
-            if (dsv.isReleased()) {
+        for (DatasetVersion dsv : ds.getVersions()) {
+            if (dsv.isReleased() || (includeDeaccessioned && dsv.isDeaccessioned())) {
                 if (dsv.getVersionNumber().equals(majorVersion) && dsv.getMinorVersionNumber().equals(minorVersion)) {
                     return dsv;
                 }
@@ -43,5 +48,4 @@ public DatasetVersion execute(CommandContext ctxt) throws CommandException {
         }
         return null;
     }
-    
-}
\ No newline at end of file
+}

From d9b3f547a14bd882e252ddc9d8060a7f40bfff3d Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 12 Sep 2023 17:24:11 +0100
Subject: [PATCH 380/922] Fixed: null check for DataFile owner in JsonPrinter

---
 .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 2b04bb3f657..e5cd72ff5fc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -691,8 +691,11 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("tabularData", df.isTabularData())
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
-                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD())
-                .add("fileAccessRequest", df.getOwner().isFileAccessRequest());
+                .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
+        Dataset dfOwner = df.getOwner();
+        if (dfOwner != null) {
+            builder.add("fileAccessRequest", dfOwner.isFileAccessRequest());
+        }
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the

From fa9d8694ca82d2399bee200d3b3aed3d6579f386 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 12 Sep 2023 12:49:30 -0400
Subject: [PATCH 381/922] merge issues

---
 .../iq/dataverse/FileAccessRequestServiceBean.java     | 10 ++++------
 .../harvard/iq/dataverse/FileDownloadServiceBean.java  |  1 +
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java
index 215e4695a75..af8577fad34 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileAccessRequestServiceBean.java
@@ -1,12 +1,10 @@
 package edu.harvard.iq.dataverse;
 
 import java.util.List;
-import javax.ejb.Stateless;
-import javax.inject.Named;
-import javax.persistence.EntityManager;
-import javax.persistence.PersistenceContext;
-
-import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import jakarta.ejb.Stateless;
+import jakarta.inject.Named;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.PersistenceContext;
 
 /**
  *
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 06509b30c88..0758f053470 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -18,6 +18,7 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import edu.harvard.iq.dataverse.util.StringUtil;

From f6fe5dcbd73bbf8d631589dc335244bd90bc0c5e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 12 Sep 2023 12:50:10 -0400
Subject: [PATCH 382/922] change gbar to Boolean to match flyway

---
 .../edu/harvard/iq/dataverse/DvObjectContainer.java | 13 +++++--------
 .../java/edu/harvard/iq/dataverse/api/Datasets.java |  4 ++--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index a18b09cb4b8..da77df786d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -39,7 +39,7 @@ public boolean isEffectivelyPermissionRoot() {
     
     private String metadataLanguage=null;
     
-    private String guestbookAtRequest = null;
+    private Boolean guestbookAtRequest = null;
     
     public String getEffectiveStorageDriverId() {
         String id = storageDriver;
@@ -101,7 +101,7 @@ public static boolean isMetadataLanguageSet(String mdLang) {
     
     public boolean getEffectiveGuestbookEntryAtRequest() {
         boolean gbAtRequest = false;
-        if (guestbookAtRequest==null || UNDEFINED_CODE.equals(guestbookAtRequest)) {
+        if (guestbookAtRequest==null) {
             if (this.getOwner() != null) {
                 gbAtRequest = this.getOwner().getEffectiveGuestbookEntryAtRequest();
             } else {
@@ -111,15 +111,12 @@ public boolean getEffectiveGuestbookEntryAtRequest() {
                 }
             }
         } else {
-            gbAtRequest = Boolean.parseBoolean(guestbookAtRequest);
+            gbAtRequest = guestbookAtRequest;
         }
         return gbAtRequest;
     }
     
-    public String getGuestbookEntryAtRequest() {
-        if (guestbookAtRequest == null) {
-            return UNDEFINED_CODE;
-        }
+    public Boolean getGuestbookEntryAtRequest() {
         return guestbookAtRequest;
     }
 
@@ -128,7 +125,7 @@ public void setGuestbookEntryAtRequest(String gbAtRequest) {
             this.guestbookAtRequest = null;
         } else {
             //Force to true or false
-            this.guestbookAtRequest = Boolean.valueOf(Boolean.parseBoolean(gbAtRequest)).toString();
+            this.guestbookAtRequest = Boolean.valueOf(Boolean.parseBoolean(gbAtRequest));
         }
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 55e329bea68..a1191a6f4e6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3888,8 +3888,8 @@ public Response getGuestbookEntryOption(@Context ContainerRequestContext crc, @P
         } catch (WrappedResponse ex) {
             return error(Response.Status.NOT_FOUND, "No such dataset");
         }
-        String gbAtRequest = dataset.getGuestbookEntryAtRequest();
-        if(gbAtRequest == null || gbAtRequest.equals(DvObjectContainer.UNDEFINED_CODE)) {
+        Boolean gbAtRequest = dataset.getGuestbookEntryAtRequest();
+        if(gbAtRequest == null) {
             return ok("Not set on dataset, using the default: " + dataset.getEffectiveGuestbookEntryAtRequest());
         }
         return ok(dataset.getEffectiveGuestbookEntryAtRequest());

From 829fed9db0333f108fb3bf01782c44ec44cf353f Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Tue, 12 Sep 2023 17:16:54 -0400
Subject: [PATCH 383/922] Fixes the issues with the permissions of the docker
 scripts on Windows reported on the issue #9904

---
 modules/container-configbaker/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile
index 564216b3572..44f3806a591 100644
--- a/modules/container-configbaker/Dockerfile
+++ b/modules/container-configbaker/Dockerfile
@@ -38,6 +38,7 @@ RUN rm ${SOLR_TEMPLATE}/conf/managed-schema.xml
 # Copy the data from scripts/api that provide the common base setup you'd get from the installer.
 # ".dockerignore" will take care of taking only the bare necessities
 COPY maven/setup ${SCRIPT_DIR}/bootstrap/base/
+RUN chmod +x ${BOOTSTRAP_DIR}/*/*.sh
 
 # Set the entrypoint to tini (as a process supervisor)
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

From bfe7f9c3537a89b75fd3190d063433c8f6147f96 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 13 Sep 2023 10:56:59 -0400
Subject: [PATCH 384/922] RestAssured tests for the new functionality added to
 the /versions api. (#9763)

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 85 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 47 +++++++++-
 2 files changed, 130 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index b353b4488d0..d5b3dbca05a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -555,6 +555,91 @@ public void testCreatePublishDestroyDataset() {
 
     }
 
+    /**
+     * The apis (/api/datasets/{id}/versions and /api/datasets/{id}/versions/{vid}
+     * are called from other RestAssured tests, in this class and also FileIT. 
+     * But this test is dedicated to this api specifically, and focuses on the 
+     * functionality added to it in 6.1. 
+    */
+    @Test
+    public void testDatasetVersionsAPI() {
+        // Create user
+        String apiToken = UtilIT.createRandomUserGetToken();
+
+        // Create user with no permission
+        String apiTokenNoPerms = UtilIT.createRandomUserGetToken();
+
+        // Create Collection
+        String collectionAlias = UtilIT.createRandomCollectionGetAlias(apiToken);
+
+        // Create Dataset
+        Response createDataset = UtilIT.createRandomDatasetViaNativeApi(collectionAlias, apiToken);
+        createDataset.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset);
+        String datasetPid = JsonPath.from(createDataset.asString()).getString("data.persistentId");
+
+        // Upload file
+        String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Integer fileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        
+        // Check that the file we just uploaded is shown by the versions api:
+        Response unpublishedDraft = UtilIT.getDatasetVersion(datasetPid, ":draft", apiToken);
+        unpublishedDraft.prettyPrint();
+        unpublishedDraft.then().assertThat()
+                .body("data.files.size()", equalTo(1))
+                .statusCode(OK.getStatusCode());
+        
+        // Now check that the file is NOT shown, when we ask the versions api to 
+        // skip files: 
+        boolean skipFiles = true; 
+        unpublishedDraft = UtilIT.getDatasetVersion(datasetPid, ":draft", apiToken, skipFiles);
+        unpublishedDraft.prettyPrint();
+        unpublishedDraft.then().assertThat()
+                .body("data.files", equalTo(null))
+                .statusCode(OK.getStatusCode());
+
+        // Publish collection and dataset
+        UtilIT.publishDataverseViaNativeApi(collectionAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+        UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        // Upload another file: 
+        String pathToFile2 = "src/main/webapp/resources/images/cc0.png";
+        Response uploadResponse2 = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile2, apiToken);
+        uploadResponse2.then().assertThat().statusCode(OK.getStatusCode());
+       
+        // We should now have a published version, and a draft. 
+        
+        // Call /versions api, *with the owner api token*, make sure both 
+        // versions are listed        
+        Response versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiToken);
+        versionsResponse.prettyPrint();
+        versionsResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.size()", equalTo(2));
+        
+        // And now call it with an un-privileged token, to make sure only one 
+        // (the published one) version is shown:
+        
+        versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiTokenNoPerms);
+        versionsResponse.prettyPrint();
+        versionsResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.size()", equalTo(1));
+
+        // And now call the "short", no-files version of the same api
+        versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiTokenNoPerms, skipFiles);
+        versionsResponse.prettyPrint();
+        versionsResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].files", equalTo(null));
+    }
+
+    
     /**
      * This test requires the root dataverse to be published to pass.
      */
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e47971f9b92..678d4e5523b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -9,6 +9,7 @@
 import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObject;
+import static jakarta.ws.rs.core.Response.Status.CREATED;
 
 import java.io.File;
 import java.io.IOException;
@@ -51,7 +52,6 @@
 import java.util.Collections;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.*;
-import static org.junit.jupiter.api.Assertions.*;
 
 public class UtilIT {
 
@@ -119,6 +119,16 @@ public static Response createRandomUser() {
 
         return createRandomUser("user");
     }
+    
+    /**
+     * A convenience method for creating a random test user, when all you need 
+     * is the api token. 
+     * @return apiToken
+     */
+    public static String createRandomUserGetToken(){
+        Response createUser = createRandomUser();        
+        return getApiTokenFromResponse(createUser);
+    }
 
     public static Response createUser(String username, String email) {
         logger.info("Creating user " + username);
@@ -369,6 +379,20 @@ static Response createRandomDataverse(String apiToken) {
         String category = null;
         return createDataverse(alias, category, apiToken);
     }
+    
+    /**
+     * A convenience method for creating a random collection and getting its 
+     * alias in one step. 
+     * @param apiToken
+     * @return alias
+     */
+    static String createRandomCollectionGetAlias(String apiToken){
+        
+        Response createCollectionResponse = createRandomDataverse(apiToken);
+        //createDataverseResponse.prettyPrint();
+        createCollectionResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        return UtilIT.getAliasFromResponse(createCollectionResponse);
+    }
 
     static Response showDataverseContents(String alias, String apiToken) {
         return given()
@@ -1403,9 +1427,17 @@ static Response nativeGetUsingPersistentId(String persistentId, String apiToken)
     }
 
     static Response getDatasetVersion(String persistentId, String versionNumber, String apiToken) {
+        return getDatasetVersion(persistentId, versionNumber, apiToken, false);
+    }
+    
+    static Response getDatasetVersion(String persistentId, String versionNumber, String apiToken, boolean skipFiles) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/datasets/:persistentId/versions/" + versionNumber + "?persistentId=" + persistentId);
+                .get("/api/datasets/:persistentId/versions/" 
+                        + versionNumber 
+                        + "?persistentId=" 
+                        + persistentId
+                        + (skipFiles ? "&includeFiles=false" : ""));
     }
 
     static Response getMetadataBlockFromDatasetVersion(String persistentId, String versionNumber, String metadataBlock, String apiToken) {
@@ -1767,6 +1799,10 @@ static Response removeDatasetThumbnail(String datasetPersistentId, String apiTok
     }
     
     static Response getDatasetVersions(String idOrPersistentId, String apiToken) {
+        return getDatasetVersions(idOrPersistentId, apiToken, false);
+    }
+    
+    static Response getDatasetVersions(String idOrPersistentId, String apiToken, boolean skipFiles) {
         logger.info("Getting Dataset Versions");
         String idInPath = idOrPersistentId; // Assume it's a number.
         String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
@@ -1774,6 +1810,13 @@ static Response getDatasetVersions(String idOrPersistentId, String apiToken) {
             idInPath = ":persistentId";
             optionalQueryParam = "?persistentId=" + idOrPersistentId;
         }
+        if (skipFiles) {
+            if ("".equals(optionalQueryParam)) {
+                optionalQueryParam = "?includeFiles=false";
+            } else {
+                optionalQueryParam = optionalQueryParam.concat("&includeFiles=false");
+            }
+        }
         RequestSpecification requestSpecification = given();
         if (apiToken != null) {
             requestSpecification = given()

From 8e894c37a17ce184bb3c59eb027dc03ed0f21274 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 13 Sep 2023 11:42:25 -0400
Subject: [PATCH 385/922] added another test, for the pagination functionality
 in the /versions api (also being added in 6.1). #9763

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 26 ++++++++++++++-----
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 22 ++++++++++++++++
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index d5b3dbca05a..4a0e1c857c7 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -557,7 +557,7 @@ public void testCreatePublishDestroyDataset() {
 
     /**
      * The apis (/api/datasets/{id}/versions and /api/datasets/{id}/versions/{vid}
-     * are called from other RestAssured tests, in this class and also FileIT. 
+     * are already called from other RestAssured tests, in this class and also FileIT. 
      * But this test is dedicated to this api specifically, and focuses on the 
      * functionality added to it in 6.1. 
     */
@@ -584,8 +584,6 @@ public void testDatasetVersionsAPI() {
         String pathToFile = "src/main/webapp/resources/images/dataverseproject.png";
         Response uploadResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
         uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        Integer fileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
         
         // Check that the file we just uploaded is shown by the versions api:
         Response unpublishedDraft = UtilIT.getDatasetVersion(datasetPid, ":draft", apiToken);
@@ -615,13 +613,27 @@ public void testDatasetVersionsAPI() {
         // We should now have a published version, and a draft. 
         
         // Call /versions api, *with the owner api token*, make sure both 
-        // versions are listed        
+        // versions are listed; also check that the correct numbers of files 
+        // are shown in each version (2 in the draft, 1 in the published). 
         Response versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiToken);
         versionsResponse.prettyPrint();
         versionsResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
-                .body("data.size()", equalTo(2));
-        
+                .body("data.size()", equalTo(2))
+                .body("data[0].files.size()", equalTo(2))
+                .body("data[1].files.size()", equalTo(1));
+        
+        // Now call the this api with the new (as of 6.1) pagination parameters
+        Integer offset = 0;
+        Integer howmany = 1;
+        versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiToken, offset, howmany);
+        // (the above should return only one version, the draft)
+        versionsResponse.prettyPrint();
+        versionsResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.size()", equalTo(1))
+                .body("data[0].files.size()", equalTo(2));
+                
         // And now call it with an un-privileged token, to make sure only one 
         // (the published one) version is shown:
         
@@ -630,7 +642,7 @@ public void testDatasetVersionsAPI() {
         versionsResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data.size()", equalTo(1));
-
+        
         // And now call the "short", no-files version of the same api
         versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiTokenNoPerms, skipFiles);
         versionsResponse.prettyPrint();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 678d4e5523b..f94cfa8e400 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -1803,6 +1803,14 @@ static Response getDatasetVersions(String idOrPersistentId, String apiToken) {
     }
     
     static Response getDatasetVersions(String idOrPersistentId, String apiToken, boolean skipFiles) {
+        return getDatasetVersions(idOrPersistentId, apiToken, null, null, skipFiles);
+    }
+    
+    static Response getDatasetVersions(String idOrPersistentId, String apiToken, Integer offset, Integer limit) {
+        return getDatasetVersions(idOrPersistentId, apiToken, offset, limit, false);
+    }
+    
+    static Response getDatasetVersions(String idOrPersistentId, String apiToken, Integer offset, Integer limit, boolean skipFiles) {
         logger.info("Getting Dataset Versions");
         String idInPath = idOrPersistentId; // Assume it's a number.
         String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
@@ -1817,6 +1825,20 @@ static Response getDatasetVersions(String idOrPersistentId, String apiToken, boo
                 optionalQueryParam = optionalQueryParam.concat("&includeFiles=false");
             }
         }
+        if (offset != null) {
+            if ("".equals(optionalQueryParam)) {
+                optionalQueryParam = "?offset="+offset;
+            } else {
+                optionalQueryParam = optionalQueryParam.concat("&offset="+offset);
+            }
+        }
+        if (limit != null) {
+            if ("".equals(optionalQueryParam)) {
+                optionalQueryParam = "?limit="+limit;
+            } else {
+                optionalQueryParam = optionalQueryParam.concat("&limit="+limit);
+            }
+        }
         RequestSpecification requestSpecification = given();
         if (apiToken != null) {
             requestSpecification = given()

From b9e99f3e7253d836aadebac8b128efa21027eef8 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 13 Sep 2023 11:43:42 -0400
Subject: [PATCH 386/922] typo in a comment. #9763

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 4a0e1c857c7..e726337cf8b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -557,7 +557,7 @@ public void testCreatePublishDestroyDataset() {
 
     /**
      * The apis (/api/datasets/{id}/versions and /api/datasets/{id}/versions/{vid}
-     * are already called from other RestAssured tests, in this class and also FileIT. 
+     * are already called from other RestAssured tests, in this class and also in FilesIT. 
      * But this test is dedicated to this api specifically, and focuses on the 
      * functionality added to it in 6.1. 
     */

From f164a681deaf14d27ee5fb35a344805d86ac631b Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 13 Sep 2023 11:46:27 -0400
Subject: [PATCH 387/922] more typos in comments. (#9763)

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index e726337cf8b..23fc5911ad0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -623,7 +623,7 @@ public void testDatasetVersionsAPI() {
                 .body("data[0].files.size()", equalTo(2))
                 .body("data[1].files.size()", equalTo(1));
         
-        // Now call the this api with the new (as of 6.1) pagination parameters
+        // Now call this api with the new (as of 6.1) pagination parameters
         Integer offset = 0;
         Integer howmany = 1;
         versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiToken, offset, howmany);
@@ -635,7 +635,7 @@ public void testDatasetVersionsAPI() {
                 .body("data[0].files.size()", equalTo(2));
                 
         // And now call it with an un-privileged token, to make sure only one 
-        // (the published one) version is shown:
+        // (the published) version is shown:
         
         versionsResponse = UtilIT.getDatasetVersions(datasetPid, apiTokenNoPerms);
         versionsResponse.prettyPrint();

From e15b80319621965acd0cc3c8aa3987f78de53442 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 10:49:29 +0200
Subject: [PATCH 388/922] feat(test): update tags for ITs in new class since
 Dataverse 6.0

With moving to JUnit5 in Dataverse 6.0, we can add nice tag strings
in a helper class. With this commit we add a tag for integration
tests ("integration"), but also add a feature like tag "testcontainers"
to allow for more granular filtering.
---
 .../oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java      | 4 +++-
 src/test/java/edu/harvard/iq/dataverse/util/testing/Tags.java | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
index 2c963e8df46..e794bfa7d48 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -16,6 +16,7 @@
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
 import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
+import edu.harvard.iq.dataverse.util.testing.Tags;
 import org.htmlunit.FailingHttpStatusCodeException;
 import org.htmlunit.WebClient;
 import org.htmlunit.WebResponse;
@@ -51,7 +52,8 @@
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.mockito.Mockito.when;
 
-@Tag("testcontainers")
+@Tag(Tags.INTEGRATION_TEST)
+@Tag(Tags.USES_TESTCONTAINERS)
 @Testcontainers
 @ExtendWith(MockitoExtension.class)
 // NOTE: order is important here - Testcontainers must be first, otherwise it's not ready when we call getAuthUrl()
diff --git a/src/test/java/edu/harvard/iq/dataverse/util/testing/Tags.java b/src/test/java/edu/harvard/iq/dataverse/util/testing/Tags.java
index dcbd0529d8b..1544d393896 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/testing/Tags.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/testing/Tags.java
@@ -2,4 +2,6 @@
 
 public class Tags {
     public static final String NOT_ESSENTIAL_UNITTESTS = "not-essential-unittests";
+    public static final String INTEGRATION_TEST = "integration";
+    public static final String USES_TESTCONTAINERS = "testcontainers";
 }

From 9f173dbd89b906bdacbeda0708d0bf5ab07e62f5 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 10:50:39 +0200
Subject: [PATCH 389/922] feat(test): make test groups (tags) in verify phase
 selectable from property

Adding a property allows creating filters on the command line.

See also: https://junit.org/junit5/docs/current/user-guide/#running-tests-tag-expressions
---
 pom.xml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 7ded2367e88..bd3c08c8256 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,6 +20,7 @@
     <properties>
         <skipUnitTests>false</skipUnitTests>
         <skipIntegrationTests>false</skipIntegrationTests>
+        <it.groups>integration</it.groups>
         
         <!-- By default, this module will produce a WAR file. -->
         <!-- This will be switched within the container profile! -->
@@ -845,7 +846,7 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>
                 <configuration>
-                    <groups>testcontainers</groups>
+                    <groups>${it.groups}</groups>
                     <argLine>${failsafe.jacoco.args} ${argLine}</argLine>
                     <skip>${skipIntegrationTests}</skip>
                 </configuration>

From 7b4153c0ceb954b5f69330ec5c3f70f4f7f2f0fc Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 10:52:22 +0200
Subject: [PATCH 390/922] feat(test): only instrument our own classes with
 JaCoCo

This way we avoid unnecessary instrumentation of dependency classes
we don't want anyway and get us in trouble when methods are too large etc.
---
 pom.xml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/pom.xml b/pom.xml
index bd3c08c8256..a8633b62b2e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -745,6 +745,11 @@
                         <configuration>
                             <destFile>${project.build.directory}/coverage-reports/jacoco-unit.exec</destFile>
                             <propertyName>surefire.jacoco.args</propertyName>
+                            <includes>
+                                <include>edu/harvard/iq/dataverse/*</include>
+                                <include>io/gdcc/*</include>
+                                <include>org/dataverse/*</include>
+                            </includes>
                         </configuration>
                     </execution>
                     <execution>
@@ -768,6 +773,11 @@
                             <destFile>${project.build.directory}/coverage-reports/jacoco-integration.exec</destFile>
                             <propertyName>failsafe.jacoco.args</propertyName>
                             <skip>${skipIntegrationTests}</skip>
+                            <includes>
+                                <include>edu/harvard/iq/dataverse/*</include>
+                                <include>io/gdcc/*</include>
+                                <include>org/dataverse/*</include>
+                            </includes>
                         </configuration>
                     </execution>
                     <execution>

From 6d1807c4751cad1f62e6fe39f4909c8ea8f75777 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 10:52:51 +0200
Subject: [PATCH 391/922] fix(test): skip JaCoCo execution also when unit tests
 are skipped

---
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pom.xml b/pom.xml
index a8633b62b2e..13a834e9a67 100644
--- a/pom.xml
+++ b/pom.xml
@@ -745,6 +745,7 @@
                         <configuration>
                             <destFile>${project.build.directory}/coverage-reports/jacoco-unit.exec</destFile>
                             <propertyName>surefire.jacoco.args</propertyName>
+                            <skip>${skipUnitTests}</skip>
                             <includes>
                                 <include>edu/harvard/iq/dataverse/*</include>
                                 <include>io/gdcc/*</include>
@@ -761,6 +762,7 @@
                         <configuration>
                             <dataFile>${project.build.directory}/coverage-reports/jacoco-unit.exec</dataFile>
                             <outputDirectory>${project.reporting.outputDirectory}/jacoco-unit-test-coverage-report</outputDirectory>
+                            <skip>${skipUnitTests}</skip>
                         </configuration>
                     </execution>
                     <execution>

From 2cf7d45381680beb31bdfec5768a26072a50eda5 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 14:12:54 +0200
Subject: [PATCH 392/922] doc(test): update testcontainers instructions after
 changes in DV 6.0

Also cleans up the mix of tabs and spaces for indentation.
---
 .../source/developers/testing.rst             | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 8e44bcc822d..18686e566f8 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -5,7 +5,7 @@ Testing
 In order to keep our codebase healthy, the Dataverse Project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing.
 
 .. contents:: |toctitle|
-	:local:
+    :local:
 
 The Health of a Codebase
 ------------------------
@@ -294,28 +294,30 @@ To run a test with Testcontainers, you will need to write a JUnit 5 test.
 Please make sure to:
 
 1. End your test class with ``IT``
-2. Provide a ``@Tag("testcontainers")`` to be picked up during testing.
+2. Annotate the test class with two tags:
 
-.. code:: java
+   .. code:: java
 
-   /** A very minimal example for a Testcontainers integration test class. */
-   @Testcontainers
-   @Tag("testcontainers")
-   class MyExampleIT { /* ... */ }
+       /** A very minimal example for a Testcontainers integration test class. */
+       @Testcontainers
+       @Tag(edu.harvard.iq.dataverse.util.testing.Tags.INTEGRATION_TEST)
+       @Tag(edu.harvard.iq.dataverse.util.testing.Tags.USES_TESTCONTAINERS)
+       class MyExampleIT { /* ... */ }
 
-If using upstream Modules, e.g. for PostgreSQL or similar, you will need to add
+If using upstream modules, e.g. for PostgreSQL or similar, you will need to add
 a dependency to ``pom.xml`` if not present. `See the PostgreSQL module example. <https://www.testcontainers.org/modules/databases/postgres/>`_
 
 To run these tests, simply call out to Maven:
 
 .. code::
 
-	 mvn -P tc verify
+    mvn verify
 
 .. note::
 
-	 1. Remember to have Docker ready to serve or tests will fail.
-	 2. This will not run any unit tests or API tests.
+    1. Remember to have Docker ready to serve or tests will fail.
+    2. You can skip running unit tests by adding ``-DskipUnitTests``
+    3. You can choose to ignore test with Testcontainers by adding ``-Dit.groups='integration & !testcontainers'``
 
 Measuring Coverage of Integration Tests
 ---------------------------------------

From d8283d9b56e9809e413fcee447971badcd8a57c5 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 14:14:56 +0200
Subject: [PATCH 393/922] doc(test): fix location of JaCoCo reports

With the combining of unit and integration test coverage reports,
the location where to access the HTML reports changed for single
and merged reports.
---
 doc/sphinx-guides/source/developers/testing.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 18686e566f8..9566cd509ca 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -114,7 +114,10 @@ Observing Changes to Code Coverage
 
 Once you've written some tests, you're probably wondering how much you've helped to increase the code coverage. In Netbeans, do a "clean and build." Then, under the "Projects" tab, right-click "dataverse" and click "Code Coverage" -> "Show Report". For each Java file you have open, you should be able to see the percentage of code that is covered by tests and every line in the file should be either green or red. Green indicates that the line is being exercised by a unit test and red indicates that it is not.
 
-In addition to seeing code coverage in Netbeans, you can also see code coverage reports by opening ``target/site/jacoco/index.html`` in your browser.
+In addition to seeing code coverage in Netbeans, you can also see code coverage reports by opening ``target/site/jacoco-X-test-coverage-report/index.html`` in your browser.
+Depending on the report type you want to look at, let ``X`` be one of ``unit``, ``integration`` or ``merged``.
+"Merged" will display combined coverage of both unit and integration test, but does currently not cover API tests.
+
 
 Testing Commands
 ^^^^^^^^^^^^^^^^

From ba87bca46a5738e55ee6e0cc7b917e8fe3fd1a33 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 14:17:56 +0200
Subject: [PATCH 394/922] doc(test): fix language for API ITs

The described testing is not about integration tests,
but end to end API tests. This commit makes the
language more precise.
---
 doc/sphinx-guides/source/developers/testing.rst | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 9566cd509ca..78e5a47fb5b 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -322,10 +322,11 @@ To run these tests, simply call out to Maven:
     2. You can skip running unit tests by adding ``-DskipUnitTests``
     3. You can choose to ignore test with Testcontainers by adding ``-Dit.groups='integration & !testcontainers'``
 
-Measuring Coverage of Integration Tests
----------------------------------------
 
-Measuring the code coverage of integration tests with Jacoco requires several steps. In order to make these steps clear we'll use "/usr/local/payara6" as the Payara directory and "dataverse" as the Payara Unix user.
+Measuring Coverage of API Tests
+-------------------------------
+
+Measuring the code coverage of API tests with Jacoco requires several steps. In order to make these steps clear we'll use "/usr/local/payara6" as the Payara directory and "dataverse" as the Payara Unix user.
 
 Please note that this was tested under Glassfish 4 but it is hoped that the same steps will work with Payara.
 
@@ -375,8 +376,8 @@ Run this as the "dataverse" user.
 
 Note that after deployment the file "/usr/local/payara6/glassfish/domains/domain1/config/jacoco.exec" exists and is empty.
 
-Run Integration Tests
-~~~~~~~~~~~~~~~~~~~~~
+Run API Tests
+~~~~~~~~~~~~~
 
 Note that even though you see "docker-aio" in the command below, we assume you are not necessarily running the test suite within Docker. (Some day we'll probably move this script to another directory.) For this reason, we pass the URL with the normal port (8080) that app servers run on to the ``run-test-suite.sh`` script.
 

From 071971981eb72f922d22a1ad9d2f1e0058b71136 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Thu, 14 Sep 2023 14:25:47 +0200
Subject: [PATCH 395/922] doc(test): add link to JUnit filter expression docs

---
 doc/sphinx-guides/source/developers/testing.rst | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 78e5a47fb5b..d7925308e4b 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -316,11 +316,12 @@ To run these tests, simply call out to Maven:
 
     mvn verify
 
-.. note::
+Notes:
 
-    1. Remember to have Docker ready to serve or tests will fail.
-    2. You can skip running unit tests by adding ``-DskipUnitTests``
-    3. You can choose to ignore test with Testcontainers by adding ``-Dit.groups='integration & !testcontainers'``
+1. Remember to have Docker ready to serve or tests will fail.
+2. You can skip running unit tests by adding ``-DskipUnitTests``
+3. You can choose to ignore test with Testcontainers by adding ``-Dit.groups='integration & !testcontainers'``
+   Learn more about `filter expressions in the JUnit 5 guide <https://junit.org/junit5/docs/current/user-guide/#running-tests-tag-expressions>`_.
 
 
 Measuring Coverage of API Tests

From efaf5d558b34705f8f6998c56a53a8a3d62050ad Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Fri, 15 Sep 2023 14:19:33 +0200
Subject: [PATCH 396/922] refactor(test,sitemap): make SiteMapUtilTest use
 better JUnit5 checks

---
 .../iq/dataverse/sitemap/SiteMapUtilTest.java | 73 +++++++++----------
 1 file changed, 35 insertions(+), 38 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java
index ac6fa1e5166..41032ffa811 100644
--- a/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtilTest.java
@@ -10,7 +10,6 @@
 import edu.harvard.iq.dataverse.util.xml.XmlValidator;
 import java.io.File;
 import java.io.IOException;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -21,17 +20,39 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
-import static org.junit.jupiter.api.Assertions.*;
+
 import static org.junit.jupiter.api.Assertions.*;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 import org.xml.sax.SAXException;
 
-public class SiteMapUtilTest {
-
+class SiteMapUtilTest {
+
+    @TempDir
+    Path tempDir;
+    Path tempDocroot;
+    
+    @BeforeEach
+    void setup() throws IOException {
+        // NOTE: This might be unsafe for parallel tests, but our @SystemProperty helper does not yet support
+        //       lookups from vars or methods.
+        System.setProperty("test.filesDir", tempDir.toString());
+        this.tempDocroot = tempDir.resolve("docroot");
+        Files.createDirectory(tempDocroot);
+    }
+    
+    @AfterEach
+    void teardown() {
+        System.clearProperty("test.filesDir");
+    }
+    
     @Test
-    public void testUpdateSiteMap() throws IOException, ParseException {
-
+    void testUpdateSiteMap() throws IOException, ParseException, SAXException {
+        // given
         List<Dataverse> dataverses = new ArrayList<>();
         String publishedDvString = "publishedDv1";
         Dataverse publishedDataverse = new Dataverse();
@@ -77,40 +98,18 @@ public void testUpdateSiteMap() throws IOException, ParseException {
         datasetVersions.add(datasetVersion);
         deaccessioned.setVersions(datasetVersions);
         datasets.add(deaccessioned);
-
-        Path tmpDirPath = Files.createTempDirectory(null);
-        String tmpDir = tmpDirPath.toString();
-        File docroot = new File(tmpDir + File.separator + "docroot");
-        docroot.mkdirs();
-        // TODO: this and the above should be replaced with JUnit 5 @TestDir
-        System.setProperty("test.filesDir", tmpDir);
-
+        
+        // when
         SiteMapUtil.updateSiteMap(dataverses, datasets);
-
-        String pathToTest = tmpDirPath + File.separator + "docroot" + File.separator + "sitemap";
-        String pathToSiteMap = pathToTest + File.separator + "sitemap.xml";
-
-        Exception wellFormedXmlException = null;
-        try {
-            assertTrue(XmlValidator.validateXmlWellFormed(pathToSiteMap));
-        } catch (Exception ex) {
-            System.out.println("Exception caught checking that XML is well formed: " + ex);
-            wellFormedXmlException = ex;
-        }
-        assertNull(wellFormedXmlException);
-
-        Exception notValidAgainstSchemaException = null;
-        try {
-            assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap, new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd")));
-        } catch (MalformedURLException | SAXException ex) {
-            System.out.println("Exception caught validating XML against the sitemap schema: " + ex);
-            notValidAgainstSchemaException = ex;
-        }
-        assertNull(notValidAgainstSchemaException);
+        
+        // then
+        String pathToSiteMap = tempDocroot.resolve("sitemap").resolve("sitemap.xml").toString();
+        assertDoesNotThrow(() -> XmlValidator.validateXmlWellFormed(pathToSiteMap));
+        assertTrue(XmlValidator.validateXmlSchema(pathToSiteMap, new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd")));
 
         File sitemapFile = new File(pathToSiteMap);
         String sitemapString = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(sitemapFile.getAbsolutePath()))));
-        System.out.println("sitemap: " + sitemapString);
+        //System.out.println("sitemap: " + sitemapString);
 
         assertTrue(sitemapString.contains("1955-11-12"));
         assertTrue(sitemapString.contains(publishedPid));
@@ -118,8 +117,6 @@ public void testUpdateSiteMap() throws IOException, ParseException {
         assertFalse(sitemapString.contains(harvestedPid));
         assertFalse(sitemapString.contains(deaccessionedPid));
 
-        System.clearProperty("test.filesDir");
-
     }
 
 }

From 573bed941cfebae0282c53785a3847cddaf6b809 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 15 Sep 2023 09:39:46 -0400
Subject: [PATCH 397/922] typo - using schema.org SoftwareApplication

---
 src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
index fd219bf9d93..1ea1a5411fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
@@ -303,7 +303,7 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) {
             
             //Start with a reference to the Dataverse software
             JsonObjectBuilder dvSoftwareBuilder = Json.createObjectBuilder()
-                    .add("@type", JsonLDTerm.ore("SoftwareApplication").getLabel())
+                    .add("@type", JsonLDTerm.schemaOrg("SoftwareApplication").getLabel())
                     .add(JsonLDTerm.schemaOrg("name").getLabel(), DATAVERSE_SOFTWARE_NAME)
                     .add(JsonLDTerm.schemaOrg("version").getLabel(), systemConfig.getVersion(true))
                     .add(JsonLDTerm.schemaOrg("url").getLabel(), DATAVERSE_SOFTWARE_URL);

From 5a7568afc9804dc5e9159dc5f395eebfc963d2c1 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 15 Sep 2023 11:12:12 -0400
Subject: [PATCH 398/922] update docs, add release note

---
 doc/release-notes/9859-ORE and Bag updates.md    | 14 ++++++++++++++
 doc/sphinx-guides/source/admin/integrations.rst  |  9 ++++++++-
 doc/sphinx-guides/source/api/native-api.rst      |  4 +++-
 doc/sphinx-guides/source/installation/config.rst | 14 +++++++-------
 4 files changed, 32 insertions(+), 9 deletions(-)
 create mode 100644 doc/release-notes/9859-ORE and Bag updates.md

diff --git a/doc/release-notes/9859-ORE and Bag updates.md b/doc/release-notes/9859-ORE and Bag updates.md
new file mode 100644
index 00000000000..dd3ae3bbbe1
--- /dev/null
+++ b/doc/release-notes/9859-ORE and Bag updates.md	
@@ -0,0 +1,14 @@
+Dataverse's OAI_ORE Metadata Export format and archival BagIT exports 
+(which include the OAI-ORE metadata export file) have been updated to include 
+information about the dataset version state, e.g. RELEASED or DEACCESSIONED 
+and to indicate which version of Dataverse was used to create the archival Bag.
+As part of the latter, the current OAI_ORE Metadata format has been given a 1.0.0 
+version designation and it is expected that any future changes to the OAI_ORE export
+format will result in a version change and that tools such as DVUploader that can
+recreate datasets from archival Bags will start indicating which version(s) of the 
+OAI_ORE format they can read.
+
+Dataverse installations that have been using archival Bags may wish to update any
+existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
+[archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
+to generate updated versions.
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst
index 21adf8338d9..9a24cf0715c 100644
--- a/doc/sphinx-guides/source/admin/integrations.rst
+++ b/doc/sphinx-guides/source/admin/integrations.rst
@@ -217,7 +217,14 @@ Sponsored by the `Ontario Council of University Libraries (OCUL) <https://ocul.o
 RDA BagIt (BagPack) Archiving
 +++++++++++++++++++++++++++++
 
-A Dataverse installation can be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant <https://www.rd-alliance.org/system/files/Research%20Data%20Repository%20Interoperability%20WG%20-%20Final%20Recommendations_reviewed_0.pdf>`_ zipped `BagIt <https://tools.ietf.org/html/draft-kunze-bagit-17>`_ bags to the `Chronopolis <https://libraries.ucsd.edu/chronopolis/>`_ via `DuraCloud <https://duraspace.org/duracloud/>`_, to a local file system, or to `Google Cloud Storage <https://cloud.google.com/storage>`_.
+A Dataverse installation can be configured to submit a copy of published Dataset versions, packaged as `Research Data Alliance conformant <https://www.rd-alliance.org/system/files/Research%20Data%20Repository%20Interoperability%20WG%20-%20Final%20Recommendations_reviewed_0.pdf>`_ zipped `BagIt <https://tools.ietf.org/html/draft-kunze-bagit-17>`_ bags to `Chronopolis <https://libraries.ucsd.edu/chronopolis/>`_ via `DuraCloud <https://duraspace.org/duracloud/>`_, a local file system, any S3 store, or to `Google Cloud Storage <https://cloud.google.com/storage>`_.
+Submission can be automated to occur upon publication, or can be done periodically (via external scripting).
+The archival status of each Dataset version can be seen in the Dataset page version table and queried via API.
+
+The archival Bags include all of the files and metadata in a given dataset version and are sufficient to recreate the dataset, e.g. in a new Dataverse instance, or potentially in another RDA-conformant repository.
+Specifically, the archival Bags include an OAI-ORE Map serialized as JSON-LD that describe the dataset and it's files, as well as information about the version of Dataverse used to export the archival Bag.
+
+The `DVUploader <https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader>`_ includes functionality to recreate a Dataset from an archival Bag produced by Dataverse (using the Dataverse API to do so).
 
 For details on how to configure this integration, see :ref:`BagIt Export` in the :doc:`/installation/config` section of the Installation Guide.
 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 4d9466703e4..e87842ab1c7 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2088,10 +2088,12 @@ The API call requires a Json body that includes the list of the fileIds that the
   curl -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" "$SERVER_URL/api/datasets/:persistentId/files/actions/:unset-embargo?persistentId=$PERSISTENT_IDENTIFIER" -d "$JSON"
   
   
+.. _Archival Status API:
+
 Get the Archival Status of a Dataset By Version
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Archiving is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to retrieve the status. Note that this requires "superuser" credentials.
+Archival :ref:`BagIt Export` is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to retrieve the status. Note that this requires "superuser" credentials.
 
 ``GET /api/datasets/$dataset-id/$version/archivalStatus`` returns the archival status of the specified dataset version.
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f9fe74afc7c..cd841e22f6c 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1,4 +1,3 @@
-=============
 Configuration
 =============
 
@@ -1425,24 +1424,25 @@ BagIt file handler configuration settings:
 BagIt Export
 ------------
 
-Your Dataverse installation may be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant <https://www.rd-alliance.org/system/files/Research%20Data%20Repository%20Interoperability%20WG%20-%20Final%20Recommendations_reviewed_0.pdf>`_ zipped `BagIt <https://tools.ietf.org/html/draft-kunze-bagit-17>`_ archival Bags (sometimes called BagPacks) to `Chronopolis <https://libraries.ucsd.edu/chronopolis/>`_ via `DuraCloud <https://duraspace.org/duracloud/>`_ or alternately to any folder on the local filesystem.
+Your Dataverse installation may be configured to submit a copy of published Datasets, packaged as `Research Data Alliance conformant <https://www.rd-alliance.org/system/files/Research%20Data%20Repository%20Interoperability%20WG%20-%20Final%20Recommendations_reviewed_0.pdf>`_ zipped `BagIt <https://tools.ietf.org/html/draft-kunze-bagit-17>`_ archival Bags (sometimes called BagPacks) to one of several supported storage services.
+Supported services include `Chronopolis <https://libraries.ucsd.edu/chronopolis/>`_ via `DuraCloud <https://duraspace.org/duracloud/>`_, Google's Cloud, and any service that can provide an S3 interface or handle files transferred to a folder on the local filesystem.
 
-These archival Bags include all of the files and metadata in a given dataset version and are sufficient to recreate the dataset, e.g. in a new Dataverse instance, or postentially in another RDA-conformant repository.
+These archival Bags include all of the files and metadata in a given dataset version and are sufficient to recreate the dataset, e.g. in a new Dataverse instance, or potentially in another RDA-conformant repository. The `DVUploader <https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader>`_ includes functionality to recreate a Dataset from an archival Bag produced by Dataverse. (Note that this functionality is distinct from the :ref:`BagIt File Handler` upload files to an existing Dataset via the Dataverse user interface.)
 
 The Dataverse Software offers an internal archive workflow which may be configured as a PostPublication workflow via an admin API call to manually submit previously published Datasets and prior versions to a configured archive such as Chronopolis. The workflow creates a `JSON-LD <http://www.openarchives.org/ore/0.9/jsonld>`_ serialized `OAI-ORE <https://www.openarchives.org/ore/>`_ map file, which is also available as a metadata export format in the Dataverse Software web interface.
 
 At present, archiving classes include the DuraCloudSubmitToArchiveCommand, LocalSubmitToArchiveCommand, GoogleCloudSubmitToArchive, and S3SubmitToArchiveCommand , which all extend the AbstractSubmitToArchiveCommand and use the configurable mechanisms discussed below. (A DRSSubmitToArchiveCommand, which works with Harvard's DRS also exists and, while specific to DRS, is a useful example of how Archivers can support single-version-only semantics and support archiving only from specified collections (with collection specific parameters)). 
 
-All current options support the archival status APIs and the same status is available in the dataset page version table (for contributors/those who could view the unpublished dataset, with more detail available to superusers).
+All current options support the :ref:`Archival Status API` calls and the same status is available in the dataset page version table (for contributors/those who could view the unpublished dataset, with more detail available to superusers).
 
 .. _Duracloud Configuration:
 
 Duracloud Configuration
 +++++++++++++++++++++++
 
-Also note that while the current Chronopolis implementation generates the archival Bag and submits it to the archive's DuraCloud interface, the step to make a 'snapshot' of the space containing the archival Bag (and verify it's successful submission) are actions a curator must take in the DuraCloud interface.
+The current Chronopolis implementation generates the archival Bag and submits it to the archive's DuraCloud interface. The step to make a 'snapshot' of the space containing the archival Bag (and verify it's successful submission) are actions a curator must take in the DuraCloud interface.
 
-The minimal configuration to support an archiver integration involves adding a minimum of two Dataverse Software Keys and any required Payara jvm options. The example instructions here are specific to the DuraCloud Archiver\:
+The minimal configuration to support archiver integration involves adding a minimum of two Dataverse Software settings. Individual archivers may require additional settings and/or Payara jvm options and micro-profile settings. The example instructions here are specific to the DuraCloud Archiver\:
 
 \:ArchiverClassName - the fully qualified class to be used for archiving. For example:
 
@@ -1452,7 +1452,7 @@ The minimal configuration to support an archiver integration involves adding a m
 
 ``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":DuraCloudHost, :DuraCloudPort, :DuraCloudContext, :BagGeneratorThreads"``
 
-The DPN archiver defines three custom settings, one of which is required (the others have defaults):
+The DuraCloud archiver defines three custom settings, one of which is required (the others have defaults):
 
 \:DuraCloudHost - the URL for your organization's Duracloud site. For example:
 

From 2ae1a9f847c256236bec8874ba307eb7e1631967 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Fri, 15 Sep 2023 17:55:26 +0200
Subject: [PATCH 399/922] Get JSON object from JsonUtil

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonParser.java  | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index febb785cd95..984c607aac7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -34,7 +34,6 @@
 import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData;
 import org.apache.commons.validator.routines.DomainValidator;
 
-import java.io.StringReader;
 import java.sql.Timestamp;
 import java.text.ParseException;
 import java.util.ArrayList;
@@ -53,7 +52,6 @@
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
-import jakarta.json.JsonReader;
 import jakarta.json.JsonString;
 import jakarta.json.JsonValue;
 import jakarta.json.JsonValue.ValueType;
@@ -682,8 +680,7 @@ private DatasetField remapGeographicCoverage(CompoundVocabularyException ex) thr
         // convert DTO to datasetField so we can back valid values.
         Gson gson = new Gson();
         String jsonString = gson.toJson(geoCoverageDTO);
-        JsonReader jsonReader = Json.createReader(new StringReader(jsonString));
-        JsonObject obj = jsonReader.readObject();
+        JsonObject obj = JsonUtil.getJsonObject(jsonString);
         DatasetField geoCoverageField = parseField(obj);
 
         // add back valid values

From cbd395e584026ea17b80577d881b4ecd62fc6dab Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 15 Sep 2023 16:53:31 -0400
Subject: [PATCH 400/922] make configure tools available at the dataset level
 #9589

---
 doc/release-notes/9589-ds-configure-tool.md   |  1 +
 .../source/admin/external-tools.rst           |  2 +-
 .../source/api/external-tools.rst             |  4 +-
 .../edu/harvard/iq/dataverse/DatasetPage.java | 16 ++++
 .../externaltools/ExternalToolHandler.java    |  7 ++
 src/main/java/propertyFiles/Bundle.properties |  1 +
 src/main/webapp/dataset.xhtml                 | 11 +++
 .../iq/dataverse/api/ExternalToolsIT.java     | 79 +++++++++++++++++++
 .../ExternalToolHandlerTest.java              | 42 ++++++++++
 9 files changed, 160 insertions(+), 3 deletions(-)
 create mode 100644 doc/release-notes/9589-ds-configure-tool.md

diff --git a/doc/release-notes/9589-ds-configure-tool.md b/doc/release-notes/9589-ds-configure-tool.md
new file mode 100644
index 00000000000..70ac5fcaa6a
--- /dev/null
+++ b/doc/release-notes/9589-ds-configure-tool.md
@@ -0,0 +1 @@
+Configure tools are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index 67075e986bb..68b5f493d49 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -115,7 +115,7 @@ Dataset level explore tools allow the user to explore all the files in a dataset
 Dataset Level Configure Tools
 +++++++++++++++++++++++++++++
 
-Configure tools at the dataset level are not currently supported.
+Dataset level configure tools can be launched by users who have edit access to the dataset and are found under the "Edit Dataset" menu.
 
 Writing Your Own External Tool
 ------------------------------
diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst
index 05affaf975e..ed68bb09ee0 100644
--- a/doc/sphinx-guides/source/api/external-tools.rst
+++ b/doc/sphinx-guides/source/api/external-tools.rst
@@ -40,7 +40,7 @@ How External Tools Are Presented to Users
 An external tool can appear in your Dataverse installation in a variety of ways:
 
 - as an explore, preview, query or configure option for a file
-- as an explore option for a dataset
+- as an explore or configure option for a dataset
 - as an embedded preview on the file landing page
 
 See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how Dataverse installations will expect to test your tool before announcing it to their users.
@@ -92,7 +92,7 @@ Terminology
 
     scope                        Whether the external tool appears and operates at the **file** level or the **dataset** level. Note that a file level tool much also specify the type of file it operates on (see "contentType" below).
 
-    types                        Whether the external tool is an **explore** tool, a **preview** tool, a **query** tool, a **configure** tool or any combination of these (multiple types are supported for a single tool). Configure tools require an API token because they make changes to data files (files within datasets). Configure tools are currently not supported at the dataset level. The older "type" keyword that allows you to pass a single type as a string is deprecated but still supported.
+    types                        Whether the external tool is an **explore** tool, a **preview** tool, a **query** tool, a **configure** tool or any combination of these (multiple types are supported for a single tool). Configure tools require an API token because they make changes to data files (files within datasets). The older "type" keyword that allows you to pass a single type as a string is deprecated but still supported.
 
     toolUrl                      The **base URL** of the tool before query parameters are added.
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index d20175b6e1a..78ccb5542ce 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -391,6 +391,7 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
     Map<Long, List<ExternalTool>> fileQueryToolsByFileId = new HashMap<>();
     List<ExternalTool> fileQueryTools = new ArrayList<>();
     private List<ExternalTool> datasetExploreTools;
+    private List<ExternalTool> datasetConfigureTools;
 
     public Boolean isHasRsyncScript() {
         return hasRsyncScript;
@@ -2153,6 +2154,7 @@ private String init(boolean initFull) {
         previewTools = externalToolService.findFileToolsByType(ExternalTool.Type.PREVIEW);
         fileQueryTools = externalToolService.findFileToolsByType(ExternalTool.Type.QUERY);
         datasetExploreTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.EXPLORE);
+        datasetConfigureTools = externalToolService.findDatasetToolsByType(ExternalTool.Type.CONFIGURE);
         rowsPerPage = 10;
         if (dataset.getId() != null && canUpdateDataset()) {
             hasRestrictedFiles = workingVersion.isHasRestrictedFile();
@@ -5572,6 +5574,10 @@ public List<ExternalTool> getDatasetExploreTools() {
         return datasetExploreTools;
     }
 
+    public List<ExternalTool> getDatasetConfigureTools() {
+        return datasetConfigureTools;
+    }
+
     Boolean thisLatestReleasedVersion = null;
 
     public boolean isThisLatestReleasedVersion() {
@@ -5789,6 +5795,16 @@ public void explore(ExternalTool externalTool) {
         PrimeFaces.current().executeScript(externalToolHandler.getExploreScript());
     }
 
+    public void configure(ExternalTool externalTool) {
+        ApiToken apiToken = null;
+        User user = session.getUser();
+        if (user instanceof AuthenticatedUser) {
+            apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
+        }
+        ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken, session.getLocaleCode());
+        PrimeFaces.current().executeScript(externalToolHandler.getConfigureScript());
+    }
+
     private FileMetadata fileMetadataForAction;
 
     public FileMetadata getFileMetadataForAction() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
index a52679deebc..de4317464e6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
@@ -253,4 +253,11 @@ public String getExploreScript() {
         logger.fine("Exploring with " + toolUrl);
         return getScriptForUrl(toolUrl);
     }
+
+    // TODO: Consider merging with getExploreScript
+    public String getConfigureScript() {
+        String toolUrl = this.getToolUrlWithQueryParams();
+        logger.fine("Configuring with " + toolUrl);
+        return getScriptForUrl(toolUrl);
+    }
 }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 997f0470cc3..837d8b2f7c1 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1367,6 +1367,7 @@ dataset.pageTitle=Add New Dataset
 dataset.accessBtn=Access Dataset
 dataset.accessBtn.header.download=Download Options
 dataset.accessBtn.header.explore=Explore Options
+dataset.accessBtn.header.configure=Configure Options
 dataset.accessBtn.header.compute=Compute Options
 dataset.accessBtn.download.size=ZIP ({0})
 dataset.accessBtn.too.big=The dataset is too large to download. Please select the files you need from the files table.
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 67dcf89c380..55bf113abec 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -453,6 +453,17 @@
                                                                 <h:outputText value="#{bundle['dataset.editBtn.itemLabel.thumbnailsAndWidgets']}"/>
                                                             </h:outputLink>
                                                         </li>
+                                                        <ui:fragment rendered="#{DatasetPage.datasetConfigureTools.size() >= 1}">
+                                                            <li class="dropdown-header">#{bundle['dataset.accessBtn.header.configure']} <span class="glyphicon glyphicon-wrench"/></li>
+                                                            <!-- Configure tool links -->
+                                                            <ui:repeat var="tool" value="#{DatasetPage.datasetConfigureTools}">
+                                                                <li>
+                                                                    <h:commandLink styleClass="btn-explore" action="#{DatasetPage.explore(tool)}">
+                                                                        <h:outputText value="#{tool.getDisplayNameLang()}"/>
+                                                                    </h:commandLink>
+                                                                </li>
+                                                            </ui:repeat>
+                                                        </ui:fragment>
                                                         <ui:fragment rendered="#{!DatasetPage.dataset.released and DatasetPage.dataset.latestVersion.versionState=='DRAFT' and permissionsWrapper.canIssueDeleteDatasetCommand(DatasetPage.dataset)}">
                                                             <li role="separator" class="divider"></li>
                                                             <li>
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
index 6f414fb3e24..a9f6055fc9e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
@@ -1,5 +1,6 @@
 package edu.harvard.iq.dataverse.api;
 
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import io.restassured.RestAssured;
 import io.restassured.path.json.JsonPath;
 import io.restassured.response.Response;
@@ -235,6 +236,84 @@ public void testDatasetLevelTool1() {
 
     }
 
+    @Test
+    public void testDatasetLevelToolConfigure() {
+
+        // Delete all external tools before testing.
+        Response getTools = UtilIT.getExternalTools();
+        getTools.prettyPrint();
+        getTools.then().assertThat()
+                .statusCode(OK.getStatusCode());
+        String body = getTools.getBody().asString();
+        JsonReader bodyObject = Json.createReader(new StringReader(body));
+        JsonArray tools = bodyObject.readObject().getJsonArray("data");
+        for (int i = 0; i < tools.size(); i++) {
+            JsonObject tool = tools.getJsonObject(i);
+            int id = tool.getInt("id");
+            Response deleteExternalTool = UtilIT.deleteExternalTool(id);
+            deleteExternalTool.prettyPrint();
+        }
+
+        Response createUser = UtilIT.createRandomUser();
+        createUser.prettyPrint();
+        createUser.then().assertThat()
+                .statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        createDataverseResponse.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDataset.prettyPrint();
+        createDataset.then().assertThat()
+                .statusCode(CREATED.getStatusCode());
+
+        Integer datasetId = JsonPath.from(createDataset.getBody().asString()).getInt("data.id");
+        String datasetPid = JsonPath.from(createDataset.getBody().asString()).getString("data.persistentId");
+
+        String toolManifest = """
+{
+   "displayName": "Turbo Dataset Config",
+   "description": "Read/write access.",
+   "types": [
+     "configure"
+   ],
+   "scope": "dataset",
+   "toolUrl": "http://datasettool1.com",
+   "toolParameters": {
+     "queryParameters": [
+       {
+         "datasetPid": "{datasetPid}"
+       },
+       {
+         "localeCode": "{localeCode}"
+       }
+     ]
+   }
+ }
+""";
+
+        Response addExternalTool = UtilIT.addExternalTool(JsonUtil.getJsonObject(toolManifest));
+        addExternalTool.prettyPrint();
+        addExternalTool.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.displayName", CoreMatchers.equalTo("Turbo Dataset Config"));
+
+        Response getExternalToolsByDatasetId = UtilIT.getExternalToolsForDataset(datasetId.toString(), "configure", apiToken);
+        getExternalToolsByDatasetId.prettyPrint();
+        getExternalToolsByDatasetId.then().assertThat()
+                .body("data[0].displayName", CoreMatchers.equalTo("Turbo Dataset Config"))
+                .body("data[0].scope", CoreMatchers.equalTo("dataset"))
+                .body("data[0].types[0]", CoreMatchers.equalTo("configure"))
+                .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://datasettool1.com?datasetPid=" + datasetPid))
+                .statusCode(OK.getStatusCode());
+
+    }
+
     @Test
     public void testAddFilelToolNoFileId() throws IOException {
         JsonObjectBuilder job = Json.createObjectBuilder();
diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
index 39bf96210fc..ad2a24ecdb8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
@@ -1,10 +1,12 @@
 package edu.harvard.iq.dataverse.externaltools;
 
+import edu.harvard.iq.dataverse.DOIServiceBean;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.FileMetadata;
+import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
@@ -15,6 +17,7 @@
 import jakarta.json.Json;
 import jakarta.json.JsonObject;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -234,4 +237,43 @@ public void testGetToolUrlWithAllowedApiCalls() {
         assertTrue(signedUrl.contains("&token="));
         System.out.println(JsonUtil.prettyPrint(jo));
     }
+
+    @Test
+    @JvmSetting(key = JvmSettings.SITE_URL, value = "https://librascholar.org")
+    public void testDatasetConfigureTool() {
+        List<ExternalToolType> externalToolTypes = new ArrayList<>();
+        var externalToolType = new ExternalToolType();
+        externalToolType.setType(ExternalTool.Type.CONFIGURE);
+        externalToolTypes.add(externalToolType);
+        var scope = ExternalTool.Scope.DATASET;
+        String toolUrl = "http://example.com";
+        var externalTool = new ExternalTool("displayName", "toolName", "description", externalToolTypes, scope, toolUrl, "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT);
+
+        externalTool.setToolParameters(Json.createObjectBuilder()
+                .add("queryParameters", Json.createArrayBuilder()
+                        .add(Json.createObjectBuilder()
+                                .add("siteUrl", "{siteUrl}")
+                        )
+                        .add(Json.createObjectBuilder()
+                                .add("datasetPid", "{datasetPid}")
+                        )
+                        .add(Json.createObjectBuilder()
+                                .add("localeCode", "{localeCode}")
+                        )
+                )
+                .build().toString());
+
+        var dataset = new Dataset();
+        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, "10.5072", "ABC123", null, DOIServiceBean.DOI_RESOLVER_URL, null));
+        ApiToken nullApiToken = null;
+        String nullLocaleCode = "en";
+        var externalToolHandler = new ExternalToolHandler(externalTool, dataset, nullApiToken, nullLocaleCode);
+        System.out.println("tool: " + externalToolHandler.getToolUrlWithQueryParams());
+        assertEquals("http://example.com?siteUrl=https://librascholar.org&datasetPid=doi:10.5072/ABC123&localeCode=en", externalToolHandler.getToolUrlWithQueryParams());
+        assertFalse(externalToolHandler.getExternalTool().isExploreTool());
+        assertEquals("configure", externalToolHandler.getExternalTool().getExternalToolTypes().get(0).getType().toString());
+        assertEquals("dataset", externalToolHandler.getExternalTool().getScope().toString());
+
+    }
+
 }

From 129985535d825ceb501cad899c6ba57771d0eee1 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Sat, 16 Sep 2023 16:31:08 +0100
Subject: [PATCH 401/922] Stash: deaccessionDataset API endpoint WIP

---
 .../harvard/iq/dataverse/api/Datasets.java    | 33 +++++++++++++++++--
 ...tLatestPublishedDatasetVersionCommand.java |  5 +--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 10 ++++++
 3 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 5064579ebfb..48d84ba95d7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -29,6 +29,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.CreateDatasetVersionCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.CreatePrivateUrlCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.CuratePublishedDatasetVersionCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.DeaccessionDatasetVersionCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetVersionCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDatasetLinkingDataverseCommand;
@@ -525,9 +526,9 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files/counts")
-    public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+    public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("includeDeaccessioned") boolean includeDeaccessioned, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         return response(req -> {
-            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, includeDeaccessioned);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
             jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
             jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion)));
@@ -3922,4 +3923,32 @@ public Response getDatasetVersionCitation(@Context ContainerRequestContext crc,
         return response(req -> ok(
                 getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getCitation(true, false)), getRequestUser(crc));
     }
+
+    @PUT
+    @AuthRequired
+    @Path("{id}/versions/{versionId}/deaccession")
+    public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, String jsonBody, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        if (":draft".equals(versionId) || ":latest".equals(versionId)) {
+            return badRequest("Only :latest-published or a specific version can be deaccessioned");
+        }
+        return response(req -> {
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, false);
+            try (StringReader stringReader = new StringReader(jsonBody)) {
+                JsonObject jsonObject = Json.createReader(stringReader).readObject();
+                datasetVersion.setVersionNote(jsonObject.getString("deaccessionReason"));
+                String deaccessionForwardURL = jsonObject.getString("deaccessionForwardURL", null);
+                if (deaccessionForwardURL != null) {
+                    try {
+                        datasetVersion.setArchiveNote(deaccessionForwardURL);
+                    } catch (IllegalArgumentException iae) {
+                        return error(Response.Status.BAD_REQUEST, "Invalid deaccession forward URL: " + iae.getMessage());
+                    }
+                }
+                execCommand(new DeaccessionDatasetVersionCommand(dvRequestService.getDataverseRequest(), datasetVersion, false));
+                return ok("Dataset " + datasetId + " deaccessioned for version " + versionId);
+            } catch (JsonParsingException jpe) {
+                return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage());
+            }
+        }, getRequestUser(crc));
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java
index 9765d0945d8..4e4252fd155 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetLatestPublishedDatasetVersionCommand.java
@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
@@ -15,7 +16,7 @@
 @RequiredPermissions({})
 public class GetLatestPublishedDatasetVersionCommand extends AbstractCommand<DatasetVersion> {
     private final Dataset ds;
-    private boolean includeDeaccessioned;
+    private final boolean includeDeaccessioned;
 
     public GetLatestPublishedDatasetVersionCommand(DataverseRequest aRequest, Dataset anAffectedDataset) {
         this(aRequest, anAffectedDataset, false);
@@ -30,7 +31,7 @@ public GetLatestPublishedDatasetVersionCommand(DataverseRequest aRequest, Datase
     @Override
     public DatasetVersion execute(CommandContext ctxt) throws CommandException {
         for (DatasetVersion dsv : ds.getVersions()) {
-            if (dsv.isReleased() || (includeDeaccessioned && dsv.isDeaccessioned())) {
+            if (dsv.isReleased() || (includeDeaccessioned && dsv.isDeaccessioned() && ctxt.permissions().requestOn(getRequest(), ds).has(Permission.EditDataset))) {
                 return dsv;
             }
         }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index d243d3c47f2..e32a813a4d3 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3386,4 +3386,14 @@ static Response getHasBeenDeleted(String dataFileId, String apiToken) {
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/files/" + dataFileId + "/hasBeenDeleted");
     }
+
+    static Response deaccessionDataset(Integer datasetId, String version, String apiToken) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("deaccessionReason", "Test deaccession.");
+        String jsonString = jsonObjectBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .put("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
+    }
 }

From ae16dadddd7978dae23dd62671c05433db2f9300 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 18 Sep 2023 10:13:52 -0400
Subject: [PATCH 402/922] minor cleanup

---
 .../iq/dataverse/globus/GlobusServiceBean.java     | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 56219f843a7..9aae4dffc03 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -130,7 +130,7 @@ ArrayList<String> checkPermisions(AccessToken clientTokenUser, String directory,
 
         return ids;
     }
-
+/*
     public void updatePermision(AccessToken clientTokenUser, String directory, String principalType, String perm)
             throws MalformedURLException {
         if (directory != null && !directory.equals("")) {
@@ -163,8 +163,8 @@ public void updatePermision(AccessToken clientTokenUser, String directory, Strin
             count++;
         }
     }
-
-    public void deletePermision(String ruleId, Logger globusLogger) throws MalformedURLException {
+*/
+    public void deletePermission(String ruleId, Logger globusLogger) throws MalformedURLException {
 
         if (ruleId.length() > 0) {
             AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
@@ -554,7 +554,9 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
                         + rawStorageId + "&fileName=" + df.getCurrentName();
             }
         }
-        return tokenUtil.replaceTokensWithValues(appUrl) + "&storePrefix=" + storePrefix;
+        String finalUrl = tokenUtil.replaceTokensWithValues(appUrl) + "&storePrefix=" + storePrefix;
+        logger.info("Calling app: " + finalUrl);
+        return finalUrl;
     }
 
     public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken) {
@@ -624,7 +626,7 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
         String taskStatus = getTaskStatus(task);
 
         if (ruleId.length() > 0) {
-            deletePermision(ruleId, globusLogger);
+            deletePermission(ruleId, globusLogger);
         }
 
         // If success, switch to an EditInProgress lock - do this before removing the
@@ -897,7 +899,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         String taskStatus = getTaskStatus(task);
 
         if (ruleId.length() > 0) {
-            deletePermision(ruleId, globusLogger);
+            deletePermission(ruleId, globusLogger);
         }
 
         if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) {

From 9562b788b7dfbfec53d6d7e9aeb52e690cddddf4 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 18 Sep 2023 10:14:43 -0400
Subject: [PATCH 403/922] start allowupload method, fix messaging when disabled

---
 .../harvard/iq/dataverse/api/Datasets.java    | 63 ++++++++++++++++++-
 src/main/java/propertyFiles/Bundle.properties |  3 +
 2 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 599890913fd..a999a71b2d4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3393,6 +3393,65 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
 
     }
 
+    /** Requests permissions for a given globus user to upload to the dataset
+     * 
+     * @param crc
+     * @param datasetId
+     * @param jsonData
+     * @return
+     * @throws IOException
+     * @throws ExecutionException
+     * @throws InterruptedException
+     */
+    @POST
+    @AuthRequired
+    @Path("{id}/allowGlobusUpload")
+    @Consumes(MediaType.APPLICATION_JSON)
+    public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, String jsonBody
+    ) throws IOException, ExecutionException, InterruptedException {
+
+
+        logger.info(" ====  (api allowGlobusUpload) jsonBody   ====== " + jsonBody);
+
+
+        if (!systemConfig.isGlobusUpload()) {
+            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
+        }
+
+        // -------------------------------------
+        // (1) Get the user from the ContainerRequestContext
+        // -------------------------------------
+        User authUser;
+        authUser = getRequestUser(crc);
+
+        // -------------------------------------
+        // (2) Get the Dataset Id
+        // -------------------------------------
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(datasetId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+
+        // Async Call
+        globusService.givePermission(jsonBody, jsonBody, jsonBody, null, datasetId, jsonBody).globusDownload(jsonData, dataset, authUser);
+
+        return ok("Async call to Globus Download started");
+
+    }
+
+    /** Monitors a globus download and removes permissions on the dir/dataset when done
+     * 
+     * @param crc
+     * @param datasetId
+     * @param jsonData
+     * @return
+     * @throws IOException
+     * @throws ExecutionException
+     * @throws InterruptedException
+     */
     @POST
     @AuthRequired
     @Path("{id}/deleteglobusRule")
@@ -3404,8 +3463,8 @@ public Response deleteglobusRule(@Context ContainerRequestContext crc, @PathPara
         logger.info(" ====  (api deleteglobusRule) jsonData   ====== " + jsonData);
 
 
-        if (!systemConfig.isHTTPUpload()) {
-            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled"));
+        if (!systemConfig.isGlobusDownload()) {
+            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
         }
 
         // -------------------------------------
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 997f0470cc3..0343e109e61 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2623,6 +2623,9 @@ datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymi
 datasets.api.creationdate=Date Created
 datasets.api.modificationdate=Last Modified Date
 datasets.api.curationstatus=Curation Status
+datasets.api.globusdownloaddisabled=File transfer from Dataverse via Globus is not available for this installation of Dataverse.
+datasets.api.globusuploaddisabled=File transfer to Dataverse via Globus is not available for this installation of Dataverse.
+
 
 
 #Dataverses.java

From f5aa17a635fb085f8c42ac31f3c73660ea5c1e5e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 18 Sep 2023 10:15:44 -0400
Subject: [PATCH 404/922] fix formatting

---
 .../java/edu/harvard/iq/dataverse/util/bagit/OREMap.java    | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
index 1ea1a5411fa..aa653a6e360 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java
@@ -327,11 +327,7 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) {
                             aggBuilder.add(JsonLDTerm.ore("aggregates").getLabel(), aggResArrayBuilder.build())
                                     .add(JsonLDTerm.schemaOrg("hasPart").getLabel(), fileArray.build()).build())
                     // and finally add the context
-                    .add("@context", contextBuilder.build())
-                    ;
-            
-            
-            
+                    .add("@context", contextBuilder.build());
             return oremapBuilder;
         }
     }

From 258023e4fd75850da3dc6a7160c26ef8711da158 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 18 Sep 2023 10:34:11 -0400
Subject: [PATCH 405/922] use cog icon to match file level #9589

---
 src/main/webapp/dataset.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 55bf113abec..b76b33a267c 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -454,7 +454,7 @@
                                                             </h:outputLink>
                                                         </li>
                                                         <ui:fragment rendered="#{DatasetPage.datasetConfigureTools.size() >= 1}">
-                                                            <li class="dropdown-header">#{bundle['dataset.accessBtn.header.configure']} <span class="glyphicon glyphicon-wrench"/></li>
+                                                            <li class="dropdown-header">#{bundle['dataset.accessBtn.header.configure']} <span class="glyphicon glyphicon-cog"/></li>
                                                             <!-- Configure tool links -->
                                                             <ui:repeat var="tool" value="#{DatasetPage.datasetConfigureTools}">
                                                                 <li>

From bf42a926c3fa72ffbc2ffbef2de5d9e253cb8511 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Mon, 18 Sep 2023 18:14:55 +0200
Subject: [PATCH 406/922] reverted out-of-band setting in S3AccessIO

---
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 12069cac650..5f7643b3115 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1834,8 +1834,7 @@ public static boolean isPackageFile(DataFile dataFile) {
     
     public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	String driverId = dataset.getEffectiveStorageDriverId();
-    	boolean directEnabled = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
+    	boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect");
     	//Should only be requested when it is allowed, but we'll log a warning otherwise
     	if(!directEnabled) {
     		logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId());

From d8ea581360b3ab4c61bbf99e5e7607396fbe99bb Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Mon, 18 Sep 2023 18:30:52 +0200
Subject: [PATCH 407/922] add-file-metadata-api made literal as the ref does
 not exist

---
 doc/sphinx-guides/source/installation/config.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index eeb87cc9ec9..8e87fb23102 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -511,7 +511,7 @@ A Dataverse installation may also be configured to reference some files (e.g. la
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
 By default, Dataverse support uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
-With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`add-file-metadata-api` call used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the add-file-metadata-api call used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 

From f1df8290df9452a307f4bc8941affb10bafb87e8 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Sep 2023 15:37:25 -0400
Subject: [PATCH 408/922] #9760 add UBC Dataverse_Utils to client libraries
 page

---
 doc/sphinx-guides/source/api/client-libraries.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index 62069f62c23..b79996d3df4 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -50,6 +50,8 @@ Python
 
 There are multiple Python modules for interacting with Dataverse APIs.
 
+`UBC's Dataverse Utilities <https://ubc-library-rc.github.io/dataverse_utils/>`_ are a set of Python console utilities which allow one to upload datasets from a tab-separated-value spreadsheet, bulk release multiple datasets, bulk delete unpublished datasets, quickly duplicate records. replace licenses, and more. For additional information see their `PyPi page <https://pypi.org/project/dataverse-utils/>`_.
+
 `EasyDataverse <https://github.com/gdcc/easyDataverse>`_ is a Python library designed to simplify the management of Dataverse datasets in an object-oriented way, giving users the ability to upload, download, and update datasets with ease. By utilizing metadata block configurations, EasyDataverse automatically generates Python objects that contain all the necessary details required to create the native Dataverse JSON format used to create or edit datasets. Adding files and directories is also possible with EasyDataverse and requires no additional API calls. This library is particularly well-suited for client applications such as workflows and scripts as it minimizes technical complexities and facilitates swift development.
 
 `pyDataverse <https://github.com/gdcc/pyDataverse>`_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger <http://stefankasberger.at>`_ from `AUSSDA - The Austrian Social Science Data Archive <https://aussda.at>`_.  

From 7cd11f3547099c382e2eaf01f10e1e00398ec588 Mon Sep 17 00:00:00 2001
From: qqmyers <jim.myers@computer.org>
Date: Mon, 18 Sep 2023 15:45:45 -0400
Subject: [PATCH 409/922] Update config.rst

Update reference
---
 doc/sphinx-guides/source/installation/config.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 8e87fb23102..8c9829e0b81 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -511,7 +511,7 @@ A Dataverse installation may also be configured to reference some files (e.g. la
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
 By default, Dataverse support uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
-With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the add-file-metadata-api call used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`Adding the Uploaded file to the Dataset <direct-add-to-dataset-api>` API call (described in the :doc:`/developers/s3-direct-upload-api` page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.
 

From 81be260e14f18d27091f2ea9e35bb20be2405989 Mon Sep 17 00:00:00 2001
From: qqmyers <jim.myers@computer.org>
Date: Mon, 18 Sep 2023 15:46:31 -0400
Subject: [PATCH 410/922] typo

---
 doc/sphinx-guides/source/installation/config.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 8c9829e0b81..c3607bd837a 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -510,7 +510,7 @@ A Dataverse installation can alternately store files in a Swift or S3-compatible
 A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
-By default, Dataverse support uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+By default, Dataverse supports uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
 With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`Adding the Uploaded file to the Dataset <direct-add-to-dataset-api>` API call (described in the :doc:`/developers/s3-direct-upload-api` page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
 The following sections describe how to set up various types of stores and how to configure for multiple stores.

From a1a1233d2dbaa09b30e306960187bf3b29fc1337 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Mon, 18 Sep 2023 15:46:43 -0400
Subject: [PATCH 411/922] #9760 move UBC beneath pyDataverse per pdurbin

---
 doc/sphinx-guides/source/api/client-libraries.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index b79996d3df4..a25efe3a5f8 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -50,12 +50,12 @@ Python
 
 There are multiple Python modules for interacting with Dataverse APIs.
 
-`UBC's Dataverse Utilities <https://ubc-library-rc.github.io/dataverse_utils/>`_ are a set of Python console utilities which allow one to upload datasets from a tab-separated-value spreadsheet, bulk release multiple datasets, bulk delete unpublished datasets, quickly duplicate records. replace licenses, and more. For additional information see their `PyPi page <https://pypi.org/project/dataverse-utils/>`_.
-
 `EasyDataverse <https://github.com/gdcc/easyDataverse>`_ is a Python library designed to simplify the management of Dataverse datasets in an object-oriented way, giving users the ability to upload, download, and update datasets with ease. By utilizing metadata block configurations, EasyDataverse automatically generates Python objects that contain all the necessary details required to create the native Dataverse JSON format used to create or edit datasets. Adding files and directories is also possible with EasyDataverse and requires no additional API calls. This library is particularly well-suited for client applications such as workflows and scripts as it minimizes technical complexities and facilitates swift development.
 
 `pyDataverse <https://github.com/gdcc/pyDataverse>`_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger <http://stefankasberger.at>`_ from `AUSSDA - The Austrian Social Science Data Archive <https://aussda.at>`_.  
 
+`UBC's Dataverse Utilities <https://ubc-library-rc.github.io/dataverse_utils/>`_ are a set of Python console utilities which allow one to upload datasets from a tab-separated-value spreadsheet, bulk release multiple datasets, bulk delete unpublished datasets, quickly duplicate records. replace licenses, and more. For additional information see their `PyPi page <https://pypi.org/project/dataverse-utils/>`_.
+
 `dataverse-client-python <https://github.com/IQSS/dataverse-client-python>`_ had its initial release in 2015. `Robert Liebowitz <https://github.com/rliebz>`_ created this library while at the `Center for Open Science (COS) <https://centerforopenscience.org>`_ and the COS uses it to integrate the `Open Science Framework (OSF) <https://osf.io>`_ with Dataverse installations via an add-on which itself is open source and listed on the :doc:`/api/apps` page.
 
 `Pooch <https://github.com/fatiando/pooch>`_ is a Python library that allows library and application developers to download data. Among other features, it takes care of various protocols, caching in OS-specific locations, checksum verification and adds optional features like progress bars or log messages. Among other popular repositories, Pooch supports Dataverse in the sense that you can reference Dataverse-hosted datasets by just a DOI and Pooch will determine the data repository type, query the Dataverse API for contained files and checksums, giving you an easy interface to download them.

From 467eb51166fe742f3506bef706f821a139e4c6b3 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 18 Sep 2023 21:46:45 +0200
Subject: [PATCH 412/922] feat(ct): add labels with service dependency versions
 to images #9928

---
 src/main/docker/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
index 88020a118b5..aa39078fb06 100644
--- a/src/main/docker/Dockerfile
+++ b/src/main/docker/Dockerfile
@@ -51,4 +51,6 @@ LABEL org.opencontainers.image.created="@git.build.time@" \
       org.opencontainers.image.vendor="Global Dataverse Community Consortium" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="Dataverse Application Image" \
-      org.opencontainers.image.description="This container image provides the research data repository software Dataverse in a box."
\ No newline at end of file
+      org.opencontainers.image.description="This container image provides the research data repository software Dataverse in a box." \
+      org.dataverse.deps.postgresql.version="@postgresql.server.version@" \
+      org.dataverse.deps.solr.version="@solr.version@"
\ No newline at end of file

From 6b3c583c6c2b730bb5ab892010da37e2889d5bbe Mon Sep 17 00:00:00 2001
From: Philip Durbin <philipdurbin@gmail.com>
Date: Mon, 18 Sep 2023 21:55:47 -0400
Subject: [PATCH 413/922] use a ref #9689

---
 doc/sphinx-guides/source/api/dataaccess.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 7b1feed7814..6a726cdc7b9 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -83,7 +83,7 @@ Basic access URI:
 
 ``/api/access/datafile/$id``
 
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to `FilePIDsEnabled <https://guides.dataverse.org/en/latest/installation/config.html?highlight=pidsenabled>`_ 
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to :ref:`:FilePIDsEnabled`. 
 
   Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
 

From 30d1bffd126a2465117bccff1d4e97269c5500e0 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 19 Sep 2023 16:22:11 +0200
Subject: [PATCH 414/922] feat(ct): make configbaker able to expose data from
 bootstrap process via env file #9933

---
 .../scripts/bootstrap.sh                      | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/modules/container-configbaker/scripts/bootstrap.sh b/modules/container-configbaker/scripts/bootstrap.sh
index 1aa9e232953..d09c1be5414 100644
--- a/modules/container-configbaker/scripts/bootstrap.sh
+++ b/modules/container-configbaker/scripts/bootstrap.sh
@@ -5,16 +5,17 @@
 set -euo pipefail
 
 function usage() {
-  echo "Usage: $(basename "$0") [-h] [-u instanceUrl] [-t timeout] [<persona>]"
+  echo "Usage: $(basename "$0") [-h] [-u instanceUrl] [-t timeout] [-e targetEnvFile] [<persona>]"
   echo ""
   echo "Execute initial configuration (bootstrapping) of an empty Dataverse instance."
   echo -n "Known personas: "
   find "${BOOTSTRAP_DIR}" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | paste -sd ' '
   echo ""
   echo "Parameters:"
-  echo "instanceUrl - Location on container network where to reach your instance. Default: 'http://dataverse:8080'"
-  echo "    timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '2m'"
-  echo "    persona - Configure persona to execute. Calls ${BOOTSTRAP_DIR}/<persona>/init.sh. Default: 'base'"
+  echo "  instanceUrl - Location on container network where to reach your instance. Default: 'http://dataverse:8080'"
+  echo "      timeout - Provide how long to wait for the instance to become available (using wait4x). Default: '2m'"
+  echo "targetEnvFile - Path to a file where the bootstrap process can expose information as env vars (e.g. dataverseAdmin's API token)"
+  echo "      persona - Configure persona to execute. Calls ${BOOTSTRAP_DIR}/<persona>/init.sh. Default: 'base'"
   echo ""
   echo "Note: This script will wait for the Dataverse instance to be available before executing the bootstrapping."
   echo "      It also checks if already bootstrapped before (availability of metadata blocks) and skip if true."
@@ -25,12 +26,14 @@ function usage() {
 # Set some defaults as documented
 DATAVERSE_URL=${DATAVERSE_URL:-"http://dataverse:8080"}
 TIMEOUT=${TIMEOUT:-"2m"}
+TARGET_ENV_FILE=${TARGET_ENV_FILE:-""}
 
-while getopts "u:t:h" OPTION
+while getopts "u:t:e:h" OPTION
 do
   case "$OPTION" in
     u) DATAVERSE_URL="$OPTARG" ;;
     t) TIMEOUT="$OPTARG" ;;
+    e) TARGET_ENV_FILE="$OPTARG" ;;
     h) usage;;
     \?) usage;;
   esac
@@ -54,6 +57,21 @@ if [[ $BLOCK_COUNT -gt 0 ]]; then
   exit 0
 fi
 
+# Provide a space to store environment variables output to
+ENV_OUT=$(mktemp)
+export ENV_OUT
+
 # Now execute the bootstrapping script
 echo "Now executing bootstrapping script at ${BOOTSTRAP_DIR}/${PERSONA}/init.sh."
-exec "${BOOTSTRAP_DIR}/${PERSONA}/init.sh"
+# shellcheck disable=SC1090
+source "${BOOTSTRAP_DIR}/${PERSONA}/init.sh"
+
+# If the env file option was given, check if the file is writeable and copy content from the temporary file
+if [[ -n "${TARGET_ENV_FILE}" ]]; then
+  if [[ -f "${TARGET_ENV_FILE}" && -w "${TARGET_ENV_FILE}" ]]; then
+    cat "${ENV_OUT}" > "${TARGET_ENV_FILE}"
+  else
+    echo "File ${TARGET_ENV_FILE} not found, is a directory or not writeable"
+    exit 2
+  fi
+fi

From 9d76d13546fca546ce06049576254da77a5b7cab Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 19 Sep 2023 16:23:13 +0200
Subject: [PATCH 415/922] feat(ct): make configbaker bootstrap dev persona
 output api token #9933

---
 modules/container-configbaker/scripts/bootstrap/dev/init.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/container-configbaker/scripts/bootstrap/dev/init.sh b/modules/container-configbaker/scripts/bootstrap/dev/init.sh
index 1042478963d..efdaee3d0c3 100644
--- a/modules/container-configbaker/scripts/bootstrap/dev/init.sh
+++ b/modules/container-configbaker/scripts/bootstrap/dev/init.sh
@@ -17,6 +17,8 @@ curl "${DATAVERSE_URL}/api/admin/settings/:DoiProvider" -X PUT -d FAKE
 
 API_TOKEN=$(grep apiToken "/tmp/setup-all.sh.out" | jq ".data.apiToken" | tr -d \")
 export API_TOKEN
+# ${ENV_OUT} comes from bootstrap.sh and will expose the saved information back to the host if enabled.
+echo "API_TOKEN=${API_TOKEN}" >> "${ENV_OUT}"
 
 echo "Publishing root dataverse..."
 curl -H "X-Dataverse-key:$API_TOKEN" -X POST "${DATAVERSE_URL}/api/dataverses/:root/actions/:publish"

From 3d1521fdb4a1cfc9b89325e9f4085e73bc3f6435 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Tue, 19 Sep 2023 14:22:15 -0400
Subject: [PATCH 416/922] #9931 document harvesting server IT config

---
 doc/sphinx-guides/source/developers/testing.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index acaeccf4f23..81e820fb869 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -225,6 +225,20 @@ If ``dataverse.siteUrl`` is absent, you can add it with:
 
 ``./asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8080"``
 
+dataverse.oai.server.maxidentifiers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The OAI Harvesting tests require that the paging limit for ListIdentifiers must be set to 2, in order to be able to trigger this paging behavior without having to create and export too many datasets:
+
+``./asadmin create-jvm-options "-Ddataverse.oai.server.maxidentifiers=2"``
+
+dataverse.oai.server.maxrecords
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The OAI Harvesting tests require that the paging limit for ListRecords must be set to 2, in order to be able to trigger this paging behavior without having to create and export too many datasets:
+
+``./asadmin create-jvm-options "-Ddataverse.oai.server.maxrecords=2"``
+
 Identifier Generation
 ^^^^^^^^^^^^^^^^^^^^^
 

From 116845c753a8364d14bad2edafcebf6a0e28dde6 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 19 Sep 2023 15:09:11 -0400
Subject: [PATCH 417/922] refactoring, add allowUpload api call

---
 .../harvard/iq/dataverse/api/Datasets.java    |   7 +-
 .../dataaccess/GlobusOverlayAccessIO.java     |   2 +-
 .../iq/dataverse/globus/GlobusEndpoint.java   |  31 ++++++
 .../dataverse/globus/GlobusServiceBean.java   | 104 ++++++++++++------
 4 files changed, 109 insertions(+), 35 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a999a71b2d4..745f294fee6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3434,11 +3434,14 @@ public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathPar
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
+        
+        JsonObject params = JsonUtil.getJsonObject(jsonBody);
+        String principal = params.getString("principal");
 
         // Async Call
-        globusService.givePermission(jsonBody, jsonBody, jsonBody, null, datasetId, jsonBody).globusDownload(jsonData, dataset, authUser);
+        globusService.givePermission("identity", principal, "rw", dataset);
 
-        return ok("Async call to Globus Download started");
+        return ok("Permission Granted");
 
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index b18e6bb7e76..965dc3c0947 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -214,7 +214,7 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
         }
     }
 
-    private static boolean isDataverseManaged(String driverId) {
+    public static boolean isDataverseManaged(String driverId) {
         return Boolean.getBoolean("dataverse.files." + driverId + ".managed");
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java
new file mode 100644
index 00000000000..d1e5d19a592
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java
@@ -0,0 +1,31 @@
+package edu.harvard.iq.dataverse.globus;
+
+public class GlobusEndpoint {
+
+    private String id;
+    private String clientToken;
+    private String basePath;
+ 
+    
+    public GlobusEndpoint(String id, String clientToken, String basePath) {
+        
+    }
+        public String getId() {
+        return id;
+    }
+        public void setId(String id) {
+        this.id = id;
+    }
+        public String getClientToken() {
+        return clientToken;
+    }
+        public void setClientToken(String clientToken) {
+        this.clientToken = clientToken;
+    }
+        public String getBasePath() {
+        return basePath;
+    }
+        public void setBasePath(String basePath) {
+        this.basePath = basePath;
+    }
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 9aae4dffc03..910ee796e0e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -46,6 +46,7 @@
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -106,23 +107,23 @@ public void setUserTransferToken(String userTransferToken) {
         this.userTransferToken = userTransferToken;
     }
 
-    ArrayList<String> checkPermisions(AccessToken clientTokenUser, String directory, String globusEndpoint,
-            String principalType, String principal) throws MalformedURLException {
-        URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/access_list");
+    private ArrayList<String> checkPermissions(GlobusEndpoint endpoint, String principalType, String principal) throws MalformedURLException {
+       
+        URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access_list");
         MakeRequestResponse result = makeRequest(url, "Bearer",
-                clientTokenUser.getOtherTokens().get(0).getAccessToken(), "GET", null);
+                endpoint.getClientToken(), "GET", null);
         ArrayList<String> ids = new ArrayList<String>();
         if (result.status == 200) {
             AccessList al = parseJson(result.jsonResponse, AccessList.class, false);
 
             for (int i = 0; i < al.getDATA().size(); i++) {
                 Permissions pr = al.getDATA().get(i);
-                if ((pr.getPath().equals(directory + "/") || pr.getPath().equals(directory))
+                if ((pr.getPath().equals(endpoint.getBasePath() + "/") || pr.getPath().equals(endpoint.getBasePath()))
                         && pr.getPrincipalType().equals(principalType)
                         && ((principal == null) || (principal != null && pr.getPrincipal().equals(principal)))) {
                     ids.add(pr.getId());
                 } else {
-                    logger.info(pr.getPath() + " === " + directory + " == " + pr.getPrincipalType());
+                    logger.info(pr.getPath() + " === " + endpoint.getBasePath() + " == " + pr.getPrincipalType());
                     continue;
                 }
             }
@@ -185,24 +186,24 @@ public void deletePermission(String ruleId, Logger globusLogger) throws Malforme
 
     }
 
-    public int givePermission(String principalType, String principal, String perm, AccessToken clientTokenUser,
-            String directory, String globusEndpoint) throws MalformedURLException {
+    public int givePermission(String principalType, String principal, String perm, Dataset dataset) throws MalformedURLException {
 
-        ArrayList<?> rules = checkPermisions(clientTokenUser, directory, globusEndpoint, principalType, principal);
+        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+        ArrayList<?> rules = checkPermissions(endpoint, principalType, principal);
 
         Permissions permissions = new Permissions();
         permissions.setDATA_TYPE("access");
         permissions.setPrincipalType(principalType);
         permissions.setPrincipal(principal);
-        permissions.setPath(directory + "/");
+        permissions.setPath(endpoint.getBasePath() + "/");
         permissions.setPermissions(perm);
 
         Gson gson = new GsonBuilder().create();
         MakeRequestResponse result = null;
         if (rules.size() == 0) {
             logger.info("Start creating the rule");
-            URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/access");
-            result = makeRequest(url, "Bearer", clientTokenUser.getOtherTokens().get(0).getAccessToken(), "POST",
+            URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access");
+            result = makeRequest(url, "Bearer", endpoint.getClientToken(), "POST",
                     gson.toJson(permissions));
 
             if (result.status == 400) {
@@ -214,9 +215,9 @@ public int givePermission(String principalType, String principal, String perm, A
             return result.status;
         } else {
             logger.info("Start Updating the rule");
-            URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/access/"
+            URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access/"
                     + rules.get(0));
-            result = makeRequest(url, "Bearer", clientTokenUser.getOtherTokens().get(0).getAccessToken(), "PUT",
+            result = makeRequest(url, "Bearer", endpoint.getClientToken(), "PUT",
                     gson.toJson(permissions));
 
             if (result.status == 400) {
@@ -438,36 +439,25 @@ static class MakeRequestResponse {
 
     }
 
-    private MakeRequestResponse findDirectory(String directory, AccessToken clientTokenUser, String globusEndpoint)
+    private MakeRequestResponse findDirectory(String directory, String clientToken, String globusEndpoint)
             throws MalformedURLException {
         URL url = new URL(" https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/ls?path="
                 + directory + "/");
 
         MakeRequestResponse result = makeRequest(url, "Bearer",
-                clientTokenUser.getOtherTokens().get(0).getAccessToken(), "GET", null);
+                clientToken, "GET", null);
         logger.info("find directory status:" + result.status);
 
         return result;
     }
 
-    public boolean giveGlobusPublicPermissions(String datasetId)
+    public boolean giveGlobusPublicPermissions(Dataset dataset)
             throws UnsupportedEncodingException, MalformedURLException {
 
-        String globusEndpoint = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusEndpoint, "");
-        String globusBasicToken = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, "");
-        if (globusEndpoint.equals("") || globusBasicToken.equals("")) {
-            return false;
-        }
-        AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
-        if (clientTokenUser == null) {
-            logger.severe("Cannot get client token ");
-            return false;
-        }
+        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
 
-        String directory = getDirectory(datasetId);
-        logger.info(directory);
 
-        MakeRequestResponse status = findDirectory(directory, clientTokenUser, globusEndpoint);
+        MakeRequestResponse status = findDirectory(endpoint.getBasePath(), endpoint.getClientToken(), endpoint.getId());
 
         if (status.status == 200) {
 
@@ -485,8 +475,7 @@ public boolean giveGlobusPublicPermissions(String datasetId)
              * 201) { logger.info("Cannot get permission for " + file.getName()); } } } }
              */
 
-            int perStatus = givePermission("all_authenticated_users", "", "r", clientTokenUser, directory,
-                    globusEndpoint);
+            int perStatus = givePermission("all_authenticated_users", "", "r", dataset);
             logger.info("givePermission status " + perStatus);
             if (perStatus == 409) {
                 logger.info("Permissions already exist or limit was reached");
@@ -1287,4 +1276,55 @@ public String calculatemime(String fileName) throws InterruptedException {
      * updatePermision(clientTokenUser, directory, "identity", "r"); return true; }
      * 
      */
+    
+    GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
+        Dataset dataset = null;
+        if (dvObject instanceof Dataset) {
+            dataset = (Dataset) dvObject;
+        } else if (dvObject instanceof DataFile) {
+            dataset = (Dataset) dvObject.getOwner();
+        } else {
+            throw new IllegalArgumentException("Unsupported DvObject type: " + dvObject.getClass().getName());
+        }
+        String driverId = dataset.getEffectiveStorageDriverId();
+        GlobusEndpoint endpoint = null;
+        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
+
+        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
+        int pathStart = endpointWithBasePath.indexOf("/");
+        logger.info("endpointWithBasePath: " + endpointWithBasePath);
+        String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
+        logger.info("directoryPath: " + directoryPath);
+
+        if (GlobusOverlayAccessIO.isDataverseManaged(driverId) && (dataset!=null)) {
+            directoryPath = directoryPath + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
+            logger.info("directoryPath now: " + directoryPath);
+
+        } else {
+            //remote store - may have path in file storageidentifier
+            String relPath = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
+            int filenameStart = relPath.lastIndexOf("/") + 1;
+            if (filenameStart > 0) {
+                directoryPath = directoryPath + relPath.substring(0, filenameStart);
+            }
+        }
+        logger.info("directoryPath finally: " + directoryPath);
+        
+        String endpointId = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
+        
+        logger.info("endpointId: " + endpointId);
+        
+        String globusToken = System.getProperty("dataverse.files." + driverId + ".globus-token");
+
+        AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
+        String clientToken = accessToken.getOtherTokens().get(0).getAccessToken();
+
+        endpoint = new GlobusEndpoint(endpointId, clientToken, directoryPath);
+
+        return endpoint;
+    }
+    
+    private static boolean isDataverseManaged(String driverId) {
+        return Boolean.getBoolean("dataverse.files." + driverId + ".managed");
+    }
 }

From e313c7cf74974de1b126ff1b13b2caf02a482fa8 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 19 Sep 2023 15:18:24 -0400
Subject: [PATCH 418/922] typo

---
 doc/release-notes/9599-guestbook-at-request.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/9599-guestbook-at-request.md b/doc/release-notes/9599-guestbook-at-request.md
index 72255150f37..e9554b71fb4 100644
--- a/doc/release-notes/9599-guestbook-at-request.md
+++ b/doc/release-notes/9599-guestbook-at-request.md
@@ -1,2 +1,2 @@
 Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
-The global default defined by this setting can be overridden at the collection level on the collection page and at the inidividual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions. 
+The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions. 

From 750069b5ac39b0f71f14507c7770f09f36ed3af2 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 19 Sep 2023 15:41:12 -0400
Subject: [PATCH 419/922] more String->Boolean changes

---
 .../java/edu/harvard/iq/dataverse/DataversePage.java   |  2 +-
 .../edu/harvard/iq/dataverse/DvObjectContainer.java    | 10 ++--------
 .../java/edu/harvard/iq/dataverse/SettingsWrapper.java | 10 +++++-----
 .../java/edu/harvard/iq/dataverse/api/Datasets.java    |  7 +++----
 4 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
index 943a74327d5..4ce7042d421 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
@@ -1286,7 +1286,7 @@ public String getCurationLabelSetNameLabel() {
         return setName;
     }
 
-    public Set<Entry<String, String>> getGuestbookEntryOptions() {
+    public Set<Entry<Boolean, String>> getGuestbookEntryOptions() {
         return settingsWrapper.getGuestbookEntryOptions(this.dataverse).entrySet();
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index da77df786d8..7c1e096dc45 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -120,15 +120,9 @@ public Boolean getGuestbookEntryAtRequest() {
         return guestbookAtRequest;
     }
 
-    public void setGuestbookEntryAtRequest(String gbAtRequest) {
-        if (gbAtRequest != null && gbAtRequest.equals(UNDEFINED_CODE)) {
-            this.guestbookAtRequest = null;
-        } else {
-            //Force to true or false
-            this.guestbookAtRequest = Boolean.valueOf(Boolean.parseBoolean(gbAtRequest));
-        }
+    public void setGuestbookEntryAtRequest(Boolean gbAtRequest) {
+        this.guestbookAtRequest = gbAtRequest;
     }
-    
 
     /* Dataverse collections can be configured to allow use of Curation labels and have this inheritable value to decide which set of labels to use.
      * This mechanism is similar to that for the storageDriver except that there is an addition option to disable use of labels. 
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index e41a18c1431..aa5680b4e14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -632,8 +632,8 @@ public String getDefaultMetadataLanguage() {
         }
     }
     
-    public Map<String, String> getGuestbookEntryOptions(DvObjectContainer target) {
-        Map<String, String> currentMap = new HashMap<String, String>();
+    public Map<Boolean, String> getGuestbookEntryOptions(DvObjectContainer target) {
+        Map<Boolean, String> currentMap = new HashMap<Boolean, String>();
         String atDownload = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atdownload");
         String atRequest = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atrequest");
         Optional<Boolean> gbDefault = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class);
@@ -649,9 +649,9 @@ public Map<String, String> getGuestbookEntryOptions(DvObjectContainer target) {
                 useDefault = (defaultOption ? atRequest : atDownload)
                         + BundleUtil.getStringFromBundle("dataverse.inherited");
             }
-            currentMap.put("null", useDefault);
-            currentMap.put(Boolean.toString(true), atRequest);
-            currentMap.put(Boolean.toString(false), atDownload);
+            currentMap.put(null, useDefault);
+            currentMap.put(Boolean.TRUE, atRequest);
+            currentMap.put(Boolean.FALSE, atDownload);
         } else {
             // Setting not defined - leave empty
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a1191a6f4e6..07ac287ddb3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3924,10 +3924,9 @@ public Response setguestbookEntryAtRequest(@Context ContainerRequestContext crc,
         if (!gbAtRequestOpt.isPresent()) {
             return error(Response.Status.FORBIDDEN, "Guestbook Entry At Request cannot be set. This server is not configured to allow it.");
         }
-        String choice = Boolean.valueOf(gbAtRequest).toString();
-        dataset.setGuestbookEntryAtRequest(choice);
+        dataset.setGuestbookEntryAtRequest(gbAtRequest);
         datasetService.merge(dataset);
-        return ok("Guestbook Entry At Request set to: " + choice);
+        return ok("Guestbook Entry At Request set to: " + gbAtRequest);
     }
     
     @DELETE
@@ -3955,7 +3954,7 @@ public Response resetGuestbookEntryAtRequest(@Context ContainerRequestContext cr
             return error(Response.Status.NOT_FOUND, "No such dataset");
         }
         
-        dataset.setGuestbookEntryAtRequest(DvObjectContainer.UNDEFINED_CODE);
+        dataset.setGuestbookEntryAtRequest(null);
         datasetService.merge(dataset);
         return ok("Guestbook Entry At Request reset to default: " + dataset.getEffectiveGuestbookEntryAtRequest());
     }

From 2e2fb380aad84d92253064ab3c58c0293b7d6c8b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 20 Sep 2023 05:53:29 +0100
Subject: [PATCH 420/922] Added: getFileDataTables endpoint permission checks
 for restricted and embargoed files

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 25 ++++++++++++++-----
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 18 +++++++++++--
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 4c411a631f1..fec60f10f3f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -18,6 +18,7 @@
 import edu.harvard.iq.dataverse.TermsOfUseAndAccessValidator;
 import edu.harvard.iq.dataverse.UserNotificationServiceBean;
 import edu.harvard.iq.dataverse.api.auth.AuthRequired;
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
@@ -79,6 +80,8 @@
 
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.jsonDT;
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
+import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
+
 import jakarta.ws.rs.core.UriInfo;
 import org.glassfish.jersey.media.multipart.FormDataBodyPart;
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -840,12 +843,22 @@ public Response getFileDownloadCount(@Context ContainerRequestContext crc, @Path
     @AuthRequired
     @Path("{id}/dataTables")
     public Response getFileDataTables(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        return response(req -> {
-            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
-            if (!dataFile.isTabularData()) {
-                return error(BAD_REQUEST, "This operation is only available for tabular files.");
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse e) {
+            return error(Response.Status.NOT_FOUND, "File not found for given id.");
+        }
+        if (dataFile.isRestricted() || FileUtil.isActivelyEmbargoed(dataFile)) {
+            DataverseRequest dataverseRequest = createDataverseRequest(getRequestUser(crc));
+            boolean hasPermissionToDownloadFile = permissionSvc.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
+            if (!hasPermissionToDownloadFile) {
+                return error(FORBIDDEN, "Insufficient permissions to access the requested information.");
             }
-            return ok(jsonDT(dataFile.getDataTables()));
-        }, getRequestUser(crc));
+        }
+        if (!dataFile.isTabularData()) {
+            return error(BAD_REQUEST, "This operation is only available for tabular files.");
+        }
+        return ok(jsonDT(dataFile.getDataTables()));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index de91e5644cf..0a16bca7008 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2190,11 +2190,25 @@ public void testGetFileDataTables() throws InterruptedException {
         // Ensure tabular file is ingested
         sleep(2000);
 
+        String testTabularFileId = Integer.toString(JsonPath.from(uploadTabularFileResponse.body().asString()).getInt("data.files[0].dataFile.id"));
+
         // Get file data tables for the tabular file and assert data is obtained
-        int testTabularFileId = JsonPath.from(uploadTabularFileResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(Integer.toString(testTabularFileId), apiToken);
+        Response getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(testTabularFileId, apiToken);
         getFileDataTablesForTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
         int dataTablesNumber = JsonPath.from(getFileDataTablesForTabularFileResponse.body().asString()).getList("data").size();
         assertTrue(dataTablesNumber > 0);
+
+        // Get file data tables for a restricted tabular file as the owner and assert data is obtained
+        Response restrictFileResponse = UtilIT.restrictFile(testTabularFileId, true, apiToken);
+        restrictFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+        getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(testTabularFileId, apiToken);
+        getFileDataTablesForTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get file data tables for a restricted tabular file as other user and assert forbidden error is thrown
+        Response createRandomUser = UtilIT.createRandomUser();
+        createRandomUser.then().assertThat().statusCode(OK.getStatusCode());
+        String randomUserApiToken = UtilIT.getApiTokenFromResponse(createRandomUser);
+        getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(testTabularFileId, randomUserApiToken);
+        getFileDataTablesForTabularFileResponse.then().assertThat().statusCode(FORBIDDEN.getStatusCode());
     }
 }

From bbfdff391f63cc412e59734b53f0992a937a594a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 20 Sep 2023 09:01:08 +0100
Subject: [PATCH 421/922] Added: deaccessionDataset API endpoint (pending IT)

---
 .../harvard/iq/dataverse/api/Datasets.java    |  2 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 83 +++++++++++++++----
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 18 +++-
 3 files changed, 82 insertions(+), 21 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 48d84ba95d7..b7d09cd5d98 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3944,7 +3944,7 @@ public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathPa
                         return error(Response.Status.BAD_REQUEST, "Invalid deaccession forward URL: " + iae.getMessage());
                     }
                 }
-                execCommand(new DeaccessionDatasetVersionCommand(dvRequestService.getDataverseRequest(), datasetVersion, false));
+                execCommand(new DeaccessionDatasetVersionCommand(req, datasetVersion, false));
                 return ok("Dataset " + datasetId + " deaccessioned for version " + versionId);
             } catch (JsonParsingException jpe) {
                 return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage());
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 6f103df3fe8..1b77e6c09e5 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3299,7 +3299,7 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, false, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3313,7 +3313,7 @@ public void getVersionFiles() throws IOException {
         String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
 
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, false, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3324,7 +3324,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(testPageSize, fileMetadatasCount);
 
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, false, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3334,7 +3334,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), false, apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3345,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), false, apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), false, apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), false, apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), false, apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,13 +3390,13 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, invalidOrderCriteria, false, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
 
         // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, apiToken);
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, false, apiToken);
 
         getVersionFilesResponseContentType.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3412,7 +3412,7 @@ public void getVersionFiles() throws IOException {
         setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory));
         setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, apiToken);
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, false, apiToken);
 
         getVersionFilesResponseCategoryName.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3427,7 +3427,7 @@ public void getVersionFiles() throws IOException {
         restrictFileResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponseRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3452,7 +3452,7 @@ public void getVersionFiles() throws IOException {
         createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3461,7 +3461,7 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
 
-        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3471,7 +3471,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Access Status Public
-        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponsePublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3483,7 +3483,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(3, fileMetadatasCount);
 
         // Test Search Text
-        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, "test_1", null, apiToken);
+        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, "test_1", null, false, apiToken);
 
         getVersionFilesResponseSearchText.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3491,6 +3491,33 @@ public void getVersionFiles() throws IOException {
 
         fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
+
+        // Test Deaccessioned
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String latestPublishedVersion = ":latest-published";
+
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, latestPublishedVersion, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // includeDeaccessioned false
+        Response getVersionFilesResponseNoDeaccessioned = UtilIT.getVersionFiles(datasetId, latestPublishedVersion, null, null, null, null, null, null, null, false, apiToken);
+        getVersionFilesResponseNoDeaccessioned.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // includeDeaccessioned true
+        Response getVersionFilesResponseDeaccessioned = UtilIT.getVersionFiles(datasetId, latestPublishedVersion, null, null, null, null, null, null, null, true, apiToken);
+        getVersionFilesResponseDeaccessioned.then().assertThat().statusCode(OK.getStatusCode());
+
+        getVersionFilesResponseDeaccessioned.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1))
+                .body("data[1].label", equalTo(testFileName2))
+                .body("data[2].label", equalTo(testFileName3))
+                .body("data[3].label", equalTo(testFileName4))
+                .body("data[4].label", equalTo(testFileName5));
     }
 
     @Test
@@ -3533,7 +3560,7 @@ public void getVersionFileCounts() throws IOException {
         createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // Getting the file counts and assert each count
-        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken);
+        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", false, apiToken);
 
         getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
 
@@ -3548,5 +3575,27 @@ public void getVersionFileCounts() throws IOException {
         assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
         assertEquals(3, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString()));
         assertEquals(1, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString()));
+
+        // Test Deaccessioned
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String latestPublishedVersion = ":latest-published";
+
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, latestPublishedVersion, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // includeDeaccessioned false
+        Response getVersionFileCountsResponseNoDeaccessioned = UtilIT.getVersionFileCounts(datasetId, latestPublishedVersion, false, apiToken);
+        getVersionFileCountsResponseNoDeaccessioned.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // includeDeaccessioned true
+        Response getVersionFileCountsResponseDeaccessioned = UtilIT.getVersionFileCounts(datasetId, latestPublishedVersion, true, apiToken);
+        getVersionFileCountsResponseDeaccessioned.then().assertThat().statusCode(OK.getStatusCode());
+
+        responseJsonPath = getVersionFileCountsResponseDeaccessioned.jsonPath();
+        assertEquals(4, (Integer) responseJsonPath.get("data.total"));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e32a813a4d3..086fef5f18a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3276,10 +3276,21 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String searchText, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId,
+                                    String version,
+                                    Integer limit,
+                                    Integer offset,
+                                    String contentType,
+                                    String accessStatus,
+                                    String categoryName,
+                                    String searchText,
+                                    String orderCriteria,
+                                    boolean includeDeaccessioned,
+                                    String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .contentType("application/json");
+                .contentType("application/json")
+                .queryParam("includeDeaccessioned", includeDeaccessioned);
         if (limit != null) {
             requestSpecification = requestSpecification.queryParam("limit", limit);
         }
@@ -3355,9 +3366,10 @@ static Response createFileEmbargo(Integer datasetId, Integer fileId, String date
                 .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
     }
 
-    static Response getVersionFileCounts(Integer datasetId, String version, String apiToken) {
+    static Response getVersionFileCounts(Integer datasetId, String version, boolean includeDeaccessioned, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .queryParam("includeDeaccessioned", includeDeaccessioned)
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
     }
 

From b19fb8267d08978b530d3be19cec7edddd72b566 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 20 Sep 2023 09:21:53 +0100
Subject: [PATCH 422/922] Added: deaccessionDataset API endpoint IT

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 1b77e6c09e5..7c0099ef34c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3598,4 +3598,41 @@ public void getVersionFileCounts() throws IOException {
         responseJsonPath = getVersionFileCountsResponseDeaccessioned.jsonPath();
         assertEquals(4, (Integer) responseJsonPath.get("data.total"));
     }
+
+    @Test
+    public void deaccessionDataset() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Test that :draft and :latest are not allowed
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":draft", apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest", apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+
+        // Test that a not found error occurs when there is no published version available
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest-published", apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // Test that the dataset is successfully deaccessioned when published
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest-published", apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Test that a not found error occurs when the only published version has already been deaccessioned
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest-published", apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+    }
 }

From b6ce32b030dded2e2dd3ebf8d2e3b8b65583ea12 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 20 Sep 2023 10:02:07 +0100
Subject: [PATCH 423/922] Refactor: dataset version string identifiers
 extracted to constants

---
 .../iq/dataverse/api/ApiConstants.java        |  5 ++
 .../harvard/iq/dataverse/api/Datasets.java    | 32 ++++-----
 .../iq/dataverse/dataset/DatasetUtil.java     |  6 +-
 .../externaltools/ExternalToolHandler.java    |  4 +-
 .../harvard/iq/dataverse/util/FileUtil.java   |  4 +-
 .../iq/dataverse/util/URLTokenUtil.java       |  5 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 71 +++++++++----------
 .../iq/dataverse/api/DownloadFilesIT.java     |  9 +--
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 12 ++--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  9 +--
 10 files changed, 84 insertions(+), 73 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java b/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java
index 296869762da..347a8946a46 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/ApiConstants.java
@@ -12,4 +12,9 @@ private ApiConstants() {
 
     // Authentication
     public static final String CONTAINER_REQUEST_CONTEXT_USER = "user";
+
+    // Dataset
+    public static final String DS_VERSION_LATEST = ":latest";
+    public static final String DS_VERSION_DRAFT = ":draft";
+    public static final String DS_VERSION_LATEST_PUBLISHED = ":latest-published";
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b7d09cd5d98..62d87b198fe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -98,6 +98,7 @@
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 
+import static edu.harvard.iq.dataverse.api.ApiConstants.*;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*;
 import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
@@ -391,8 +392,8 @@ public Response destroyDataset(@Context ContainerRequestContext crc, @PathParam(
     @AuthRequired
     @Path("{id}/versions/{versionId}")
     public Response deleteDraftVersion(@Context ContainerRequestContext crc, @PathParam("id") String id,  @PathParam("versionId") String versionId ){
-        if ( ! ":draft".equals(versionId) ) {
-            return badRequest("Only the :draft version can be deleted");
+        if (!DS_VERSION_DRAFT.equals(versionId)) {
+            return badRequest("Only the " + DS_VERSION_DRAFT + " version can be deleted");
         }
 
         return response( req -> {
@@ -545,7 +546,7 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc, @Path
     public Response getFileAccessFolderView(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @QueryParam("version") String versionId, @QueryParam("folder") String folderName, @QueryParam("original") Boolean originals, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) {
 
         folderName = folderName == null ? "" : folderName;
-        versionId = versionId == null ? ":latest-published" : versionId;
+        versionId = versionId == null ? DS_VERSION_LATEST_PUBLISHED : versionId;
         
         DatasetVersion version;
         try {
@@ -620,8 +621,8 @@ public Response getVersionMetadataBlock(@Context ContainerRequestContext crc,
     @AuthRequired
     @Path("{id}/versions/{versionId}/linkset")
     public Response getLinkset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        if ( ":draft".equals(versionId) ) {
-            return badRequest("Signposting is not supported on the :draft version");
+        if (DS_VERSION_DRAFT.equals(versionId)) {
+            return badRequest("Signposting is not supported on the " + DS_VERSION_DRAFT + " version");
         }
         User user = getRequestUser(crc);
         return response(req -> {
@@ -706,10 +707,9 @@ public Response updateDatasetPIDMetadataAll(@Context ContainerRequestContext crc
     @AuthRequired
     @Path("{id}/versions/{versionId}")
     @Consumes(MediaType.APPLICATION_JSON)
-    public Response updateDraftVersion(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id,  @PathParam("versionId") String versionId){
-      
-        if ( ! ":draft".equals(versionId) ) {
-            return error( Response.Status.BAD_REQUEST, "Only the :draft version can be updated");
+    public Response updateDraftVersion(@Context ContainerRequestContext crc, String jsonBody, @PathParam("id") String id, @PathParam("versionId") String versionId) {
+        if (!DS_VERSION_DRAFT.equals(versionId)) {
+            return error( Response.Status.BAD_REQUEST, "Only the " + DS_VERSION_DRAFT + " version can be updated");
         }
         
         try ( StringReader rdr = new StringReader(jsonBody) ) {
@@ -792,7 +792,7 @@ public Response getVersionJsonLDMetadata(@Context ContainerRequestContext crc, @
     @Path("{id}/metadata")
     @Produces("application/ld+json, application/json-ld")
     public Response getVersionJsonLDMetadata(@Context ContainerRequestContext crc, @PathParam("id") String id, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return getVersionJsonLDMetadata(crc, id, ":draft", uriInfo, headers);
+        return getVersionJsonLDMetadata(crc, id, DS_VERSION_DRAFT, uriInfo, headers);
     }
 
     @PUT
@@ -1726,7 +1726,7 @@ public Response getCustomTermsTab(@PathParam("id") String id, @PathParam("versio
                 return error(Status.NOT_FOUND, "This Dataset has no custom license");
             }
             persistentId = getRequestParameter(":persistentId".substring(1));
-            if (versionId.equals(":draft")) {
+            if (versionId.equals(DS_VERSION_DRAFT)) {
                 versionId = "DRAFT";
             }
         } catch (WrappedResponse wrappedResponse) {
@@ -2687,11 +2687,11 @@ private void msgt(String m) {
     public static <T> T handleVersion(String versionId, DsVersionHandler<T> hdl)
             throws WrappedResponse {
         switch (versionId) {
-            case ":latest":
+            case DS_VERSION_LATEST:
                 return hdl.handleLatest();
-            case ":draft":
+            case DS_VERSION_DRAFT:
                 return hdl.handleDraft();
-            case ":latest-published":
+            case DS_VERSION_LATEST_PUBLISHED:
                 return hdl.handleLatestPublished();
             default:
                 try {
@@ -3928,8 +3928,8 @@ public Response getDatasetVersionCitation(@Context ContainerRequestContext crc,
     @AuthRequired
     @Path("{id}/versions/{versionId}/deaccession")
     public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, String jsonBody, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        if (":draft".equals(versionId) || ":latest".equals(versionId)) {
-            return badRequest("Only :latest-published or a specific version can be deaccessioned");
+        if (DS_VERSION_DRAFT.equals(versionId) || DS_VERSION_LATEST.equals(versionId)) {
+            return badRequest("Only " + DS_VERSION_LATEST_PUBLISHED + " or a specific version can be deaccessioned");
         }
         return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, false);
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
index adbd132bce8..ac1567b24e5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
@@ -9,6 +9,8 @@
 import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT;
 import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
@@ -580,10 +582,10 @@ public static String getLicenseURI(DatasetVersion dsv) {
         // Return the URI
         // For standard licenses, just return the stored URI
         return (license != null) ? license.getUri().toString()
-                // For custom terms, construct a URI with :draft or the version number in the URI
+                // For custom terms, construct a URI with draft version constant or the version number in the URI
                 : (dsv.getVersionState().name().equals("DRAFT")
                         ? dsv.getDataverseSiteUrl()
-                                + "/api/datasets/:persistentId/versions/:draft/customlicense?persistentId="
+                                + "/api/datasets/:persistentId/versions/" + DS_VERSION_DRAFT + "/customlicense?persistentId="
                                 + dsv.getDataset().getGlobalId().asString()
                         : dsv.getDataverseSiteUrl() + "/api/datasets/:persistentId/versions/" + dsv.getVersionNumber()
                                 + "." + dsv.getMinorVersionNumber() + "/customlicense?persistentId="
diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
index a52679deebc..570ef7d4194 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
@@ -34,6 +34,8 @@
 
 import org.apache.commons.codec.binary.StringUtils;
 
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_LATEST;
+
 /**
  * Handles an operation on a specific file. Requires a file id in order to be
  * instantiated. Applies logic based on an {@link ExternalTool} specification,
@@ -110,7 +112,7 @@ public String handleRequest(boolean preview) {
                 switch (externalTool.getScope()) {
                 case DATASET:
                     callback=SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/"
-                            + dataset.getId() + "/versions/:latest/toolparams/" + externalTool.getId();
+                            + dataset.getId() + "/versions/" + DS_VERSION_LATEST + "/toolparams/" + externalTool.getId();
                     break;
                 case FILE:
                     callback= SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/files/"
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 5f7643b3115..327609d5e47 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -34,6 +34,8 @@
 import edu.harvard.iq.dataverse.dataset.DatasetThumbnail;
 import edu.harvard.iq.dataverse.dataset.DatasetUtil;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
+
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT;
 import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
 import edu.harvard.iq.dataverse.ingest.IngestReport;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
@@ -2152,7 +2154,7 @@ private static String getFileAccessUrl(FileMetadata fileMetadata, String apiLoca
     private static String getFolderAccessUrl(DatasetVersion version, String currentFolder, String subFolder, String apiLocation, boolean originals) {
         String datasetId = version.getDataset().getId().toString();
         String versionTag = version.getFriendlyVersionNumber();
-        versionTag = versionTag.replace("DRAFT", ":draft");
+        versionTag = versionTag.replace("DRAFT", DS_VERSION_DRAFT);
         if (!"".equals(currentFolder)) {
             subFolder = currentFolder + "/" + subFolder;
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java
index 4ae76a7b8db..c864823176e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java
@@ -14,6 +14,8 @@
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT;
+
 public class URLTokenUtil {
 
     protected static final Logger logger = Logger.getLogger(URLTokenUtil.class.getCanonicalName());
@@ -177,8 +179,7 @@ private String getTokenValue(String value) {
                 }
             }
             if (("DRAFT").equals(versionString)) {
-                versionString = ":draft"; // send the token needed in api calls that can be substituted for a numeric
-                                          // version.
+                versionString = DS_VERSION_DRAFT; // send the token needed in api calls that can be substituted for a numeric version.
             }
             return versionString;
         case FILE_METADATA_ID:
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 7c0099ef34c..5c1eb66b63d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3,6 +3,7 @@
 import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
 import io.restassured.RestAssured;
 
+import static edu.harvard.iq.dataverse.api.ApiConstants.*;
 import static io.restassured.RestAssured.given;
 
 import io.restassured.path.json.JsonPath;
@@ -500,7 +501,7 @@ public void testCreatePublishDestroyDataset() {
         assertTrue(datasetContactFromExport.toString().contains("finch@mailinator.com"));
         assertTrue(firstValue.toString().contains("finch@mailinator.com"));
 
-        Response getDatasetVersion = UtilIT.getDatasetVersion(datasetPersistentId, ":latest-published", apiToken);
+        Response getDatasetVersion = UtilIT.getDatasetVersion(datasetPersistentId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         getDatasetVersion.prettyPrint();
         getDatasetVersion.then().assertThat()
                 .body("data.datasetId", equalTo(datasetId))
@@ -1159,7 +1160,7 @@ public void testPrivateUrl() {
         assertEquals(OK.getStatusCode(), createPrivateUrlForPostVersionOneDraft.getStatusCode());
 
         // A Contributor has DeleteDatasetDraft
-        Response deleteDraftVersionAsContributor = UtilIT.deleteDatasetVersionViaNativeApi(datasetId, ":draft", contributorApiToken);
+        Response deleteDraftVersionAsContributor = UtilIT.deleteDatasetVersionViaNativeApi(datasetId, DS_VERSION_DRAFT, contributorApiToken);
         deleteDraftVersionAsContributor.prettyPrint();
         deleteDraftVersionAsContributor.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3257,7 +3258,7 @@ public void getDatasetVersionCitation() {
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
-        Response getDatasetVersionCitationResponse = UtilIT.getDatasetVersionCitation(datasetId, ":draft", apiToken);
+        Response getDatasetVersionCitationResponse = UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_DRAFT, apiToken);
         getDatasetVersionCitationResponse.prettyPrint();
 
         getDatasetVersionCitationResponse.then().assertThat()
@@ -3293,13 +3294,11 @@ public void getVersionFiles() throws IOException {
         UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName5, new byte[300], apiToken);
         UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName4, new byte[400], apiToken);
 
-        String testDatasetVersion = ":latest";
-
         // Test pagination and NameAZ order criteria (the default criteria)
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, false, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, testPageSize, null, null, null, null, null, null, false, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3313,7 +3312,7 @@ public void getVersionFiles() throws IOException {
         String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
 
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, false, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, testPageSize, testPageSize, null, null, null, null, null, false, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3324,7 +3323,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(testPageSize, fileMetadatasCount);
 
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, false, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, testPageSize, testPageSize * 2, null, null, null, null, null, false, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3334,7 +3333,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), false, apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), false, apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3344,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), false, apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), false, apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3355,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), false, apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), false, apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3366,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), false, apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), false, apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3377,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), false, apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), false, apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,13 +3389,13 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, invalidOrderCriteria, false, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, null, invalidOrderCriteria, false, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
 
         // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, false, apiToken);
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, "image/png", null, null, null, null, false, apiToken);
 
         getVersionFilesResponseContentType.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3412,7 +3411,7 @@ public void getVersionFiles() throws IOException {
         setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory));
         setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, false, apiToken);
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, testCategory, null, null, false, apiToken);
 
         getVersionFilesResponseCategoryName.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3427,7 +3426,7 @@ public void getVersionFiles() throws IOException {
         restrictFileResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, false, apiToken);
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponseRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3452,7 +3451,7 @@ public void getVersionFiles() throws IOException {
         createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, false, apiToken);
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3461,7 +3460,7 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
 
-        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, false, apiToken);
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3471,7 +3470,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Access Status Public
-        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, false, apiToken);
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, false, apiToken);
 
         getVersionFilesResponsePublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3483,7 +3482,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(3, fileMetadatasCount);
 
         // Test Search Text
-        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, "test_1", null, false, apiToken);
+        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST, null, null, null, null, null, "test_1", null, false, apiToken);
 
         getVersionFilesResponseSearchText.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3498,17 +3497,15 @@ public void getVersionFiles() throws IOException {
         Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
         publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        String latestPublishedVersion = ":latest-published";
-
-        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, latestPublishedVersion, apiToken);
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // includeDeaccessioned false
-        Response getVersionFilesResponseNoDeaccessioned = UtilIT.getVersionFiles(datasetId, latestPublishedVersion, null, null, null, null, null, null, null, false, apiToken);
+        Response getVersionFilesResponseNoDeaccessioned = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST_PUBLISHED, null, null, null, null, null, null, null, false, apiToken);
         getVersionFilesResponseNoDeaccessioned.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
 
         // includeDeaccessioned true
-        Response getVersionFilesResponseDeaccessioned = UtilIT.getVersionFiles(datasetId, latestPublishedVersion, null, null, null, null, null, null, null, true, apiToken);
+        Response getVersionFilesResponseDeaccessioned = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST_PUBLISHED, null, null, null, null, null, null, null, true, apiToken);
         getVersionFilesResponseDeaccessioned.then().assertThat().statusCode(OK.getStatusCode());
 
         getVersionFilesResponseDeaccessioned.then().assertThat()
@@ -3560,7 +3557,7 @@ public void getVersionFileCounts() throws IOException {
         createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // Getting the file counts and assert each count
-        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", false, apiToken);
+        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST, false, apiToken);
 
         getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
 
@@ -3582,17 +3579,15 @@ public void getVersionFileCounts() throws IOException {
         Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
         publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        String latestPublishedVersion = ":latest-published";
-
-        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, latestPublishedVersion, apiToken);
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // includeDeaccessioned false
-        Response getVersionFileCountsResponseNoDeaccessioned = UtilIT.getVersionFileCounts(datasetId, latestPublishedVersion, false, apiToken);
+        Response getVersionFileCountsResponseNoDeaccessioned = UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST_PUBLISHED, false, apiToken);
         getVersionFileCountsResponseNoDeaccessioned.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
 
         // includeDeaccessioned true
-        Response getVersionFileCountsResponseDeaccessioned = UtilIT.getVersionFileCounts(datasetId, latestPublishedVersion, true, apiToken);
+        Response getVersionFileCountsResponseDeaccessioned = UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST_PUBLISHED, true, apiToken);
         getVersionFileCountsResponseDeaccessioned.then().assertThat().statusCode(OK.getStatusCode());
 
         responseJsonPath = getVersionFileCountsResponseDeaccessioned.jsonPath();
@@ -3613,14 +3608,14 @@ public void deaccessionDataset() {
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
-        // Test that :draft and :latest are not allowed
-        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":draft", apiToken);
+        // Test that draft and latest version constants are not allowed
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_DRAFT, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest", apiToken);
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
 
         // Test that a not found error occurs when there is no published version available
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest-published", apiToken);
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
 
         // Test that the dataset is successfully deaccessioned when published
@@ -3628,11 +3623,11 @@ public void deaccessionDataset() {
         publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
         Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
         publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest-published", apiToken);
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // Test that a not found error occurs when the only published version has already been deaccessioned
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, ":latest-published", apiToken);
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java
index 598ba36c1e1..927efb0b142 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DownloadFilesIT.java
@@ -16,6 +16,9 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
+
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT;
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_LATEST_PUBLISHED;
 import static jakarta.ws.rs.core.Response.Status.CREATED;
 import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
 import static jakarta.ws.rs.core.Response.Status.OK;
@@ -188,8 +191,7 @@ public void downloadAllFilesByVersion() throws IOException {
         HashSet<String> expectedFiles6 = new HashSet<>(Arrays.asList("CODE_OF_CONDUCT.md", "LICENSE.md", "MANIFEST.TXT", "README.md", "CONTRIBUTING.md"));
         assertEquals(expectedFiles6, filenamesFound6);
 
-        String datasetVersionLatestPublished = ":latest-published";
-        Response downloadFiles9 = UtilIT.downloadFiles(datasetPid, datasetVersionLatestPublished, apiToken);
+        Response downloadFiles9 = UtilIT.downloadFiles(datasetPid, DS_VERSION_LATEST_PUBLISHED, apiToken);
         downloadFiles9.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
@@ -200,8 +202,7 @@ public void downloadAllFilesByVersion() throws IOException {
         assertEquals(expectedFiles7, filenamesFound7);
 
         // Guests cannot download draft versions.
-        String datasetVersionDraft = ":draft";
-        Response downloadFiles10 = UtilIT.downloadFiles(datasetPid, datasetVersionDraft, null);
+        Response downloadFiles10 = UtilIT.downloadFiles(datasetPid, DS_VERSION_DRAFT, null);
         downloadFiles10.prettyPrint();
         downloadFiles10.then().assertThat()
                 .statusCode(UNAUTHORIZED.getStatusCode())
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 7f1ca4c8d70..94e895a7b7b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -10,6 +10,8 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.BeforeAll;
 import io.restassured.path.json.JsonPath;
+
+import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT;
 import static io.restassured.path.json.JsonPath.with;
 import io.restassured.path.xml.XmlPath;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -1354,7 +1356,7 @@ public void testDataSizeInDataverse() throws InterruptedException {
                 .statusCode(OK.getStatusCode());
         String apiTokenRando = createUserGetToken();
         
-        Response datasetStorageSizeResponseDraft = UtilIT.findDatasetDownloadSize(datasetId.toString(), ":draft", apiTokenRando);
+        Response datasetStorageSizeResponseDraft = UtilIT.findDatasetDownloadSize(datasetId.toString(), DS_VERSION_DRAFT, apiTokenRando);
         datasetStorageSizeResponseDraft.prettyPrint();
         assertEquals(UNAUTHORIZED.getStatusCode(), datasetStorageSizeResponseDraft.getStatusCode());  
         Response publishDatasetResp = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
@@ -1607,7 +1609,7 @@ public void test_CrawlableAccessToDatasetFiles() {
         // Expected values in the output: 
         String expectedTitleTopFolder = "Index of folder /";
         String expectedLinkTopFolder = folderName + "/";
-        String expectedLinkAhrefTopFolder = "/api/datasets/"+datasetId+"/dirindex/?version=:draft&folder=subfolder";
+        String expectedLinkAhrefTopFolder = "/api/datasets/"+datasetId+"/dirindex/?version=" + DS_VERSION_DRAFT + "&folder=subfolder";
         
         String expectedTitleSubFolder = "Index of folder /" + folderName;
         String expectedLinkAhrefSubFolder = "/api/access/datafile/" + folderName + "/" + dataFileId;
@@ -1987,7 +1989,7 @@ public void testDeleteFile() {
         deleteResponse2.then().assertThat().statusCode(OK.getStatusCode());
 
         // Check file 2 deleted from post v1.0 draft
-        Response postv1draft = UtilIT.getDatasetVersion(datasetPid, ":draft", apiToken);
+        Response postv1draft = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken);
         postv1draft.prettyPrint();
         postv1draft.then().assertThat()
                 .body("data.files.size()", equalTo(1))
@@ -2009,7 +2011,7 @@ public void testDeleteFile() {
         downloadResponse2.then().assertThat().statusCode(OK.getStatusCode());
 
         // Check file 3 still in post v1.0 draft
-        Response postv1draft2 = UtilIT.getDatasetVersion(datasetPid, ":draft", apiToken);
+        Response postv1draft2 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken);
         postv1draft2.prettyPrint();
         postv1draft2.then().assertThat()
                 .body("data.files[0].dataFile.filename", equalTo("orcid_16x16.png"))
@@ -2024,7 +2026,7 @@ public void testDeleteFile() {
         deleteResponse3.then().assertThat().statusCode(OK.getStatusCode());
 
         // Check file 3 deleted from post v1.0 draft
-        Response postv1draft3 = UtilIT.getDatasetVersion(datasetPid, ":draft", apiToken);
+        Response postv1draft3 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken);
         postv1draft3.prettyPrint();
         postv1draft3.then().assertThat()
                 .body("data.files[0]", equalTo(null))
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 086fef5f18a..8c6a2d6e75d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -38,6 +38,7 @@
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 
+import static edu.harvard.iq.dataverse.api.ApiConstants.*;
 import static io.restassured.path.xml.XmlPath.from;
 import static io.restassured.RestAssured.given;
 import edu.harvard.iq.dataverse.DatasetField;
@@ -515,7 +516,7 @@ static Response updateDatasetMetadataViaNative(String persistentId, String pathT
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .body(jsonIn)
                 .contentType("application/json")
-                .put("/api/datasets/:persistentId/versions/:draft?persistentId=" + persistentId);
+                .put("/api/datasets/:persistentId/versions/" + DS_VERSION_DRAFT + "?persistentId=" + persistentId);
         return response;
     }
     
@@ -791,7 +792,7 @@ static Response deleteAuxFile(Long fileId, String formatTag, String formatVersio
     static Response getCrawlableFileAccess(String datasetId, String folderName, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken);
-        String apiPath = "/api/datasets/" + datasetId + "/dirindex?version=:draft";
+        String apiPath = "/api/datasets/" + datasetId + "/dirindex?version=" + DS_VERSION_DRAFT;
         if (StringUtil.nonEmpty(folderName)) {
             apiPath = apiPath.concat("&folder="+folderName);
         }
@@ -1407,7 +1408,7 @@ static Response getDatasetVersion(String persistentId, String versionNumber, Str
     static Response getMetadataBlockFromDatasetVersion(String persistentId, String versionNumber, String metadataBlock, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/datasets/:persistentId/versions/:latest-published/metadata/citation?persistentId=" + persistentId);
+                .get("/api/datasets/:persistentId/versions/" + DS_VERSION_LATEST_PUBLISHED + "/metadata/citation?persistentId=" + persistentId);
     }
 
     static Response makeSuperUser(String username) {
@@ -2922,7 +2923,7 @@ static Response findDatasetStorageSize(String datasetId, String apiToken) {
     
     static Response findDatasetDownloadSize(String datasetId) {
         return given()
-                .get("/api/datasets/" + datasetId + "/versions/:latest/downloadsize");
+                .get("/api/datasets/" + datasetId + "/versions/" + DS_VERSION_LATEST + "/downloadsize");
     }
     
     static Response findDatasetDownloadSize(String datasetId, String version,  String apiToken) {

From 35a5278402a13075017c3c98cea0a17d7f2da167 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 20 Sep 2023 12:22:59 -0400
Subject: [PATCH 424/922] cleanup

---
 .../edu/harvard/iq/dataverse/DatasetPage.java |  2 +-
 .../iq/dataverse/ManagePermissionsPage.java   | 45 -------------------
 2 files changed, 1 insertion(+), 46 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index fcad6ae081a..5ce6207250d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3280,7 +3280,7 @@ private boolean filterSelectedFiles(){
         }
         return someFiles;
     }
-//QDRADA - still needed?
+
     public void validateFilesForRequestAccess(){
         this.filterSelectedFiles();
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
index 9bb00bbb6d5..0e277c5aa32 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ManagePermissionsPage.java
@@ -401,51 +401,6 @@ public List<RoleAssignee> completeRoleAssignee( String query ) {
         return roleAssigneeService.filterRoleAssignees(query, dvObject, roleAssignSelectedRoleAssignees);
     }
 
-    public void grantAccess(ActionEvent evt) {
-        //QDRADA 
-        logger.info("grantAccess Called");
-        try {
-        throw new Exception("grantAccessCalled");
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        /*
-        // Find the built in file downloader role (currently by alias)
-        DataverseRole fileDownloaderRole = roleService.findBuiltinRoleByAlias(DataverseRole.FILE_DOWNLOADER);
-        for (RoleAssignee roleAssignee : selectedRoleAssignees) {
-            boolean sendNotification = false;
-            for (DataFile file : selectedFiles) {
-                if (assignRole(roleAssignee, file, fileDownloaderRole)) {
-                    if (file.isReleased()) {
-                        sendNotification = true;
-                    }
-                    // remove request, if it exist
-                    for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
-                            if (file.getFileAccessRequesters().remove(au)) {
-                                List<FileAccessRequest> fileAccessRequests = fileAccessRequestService.findAllByAuthenticatedUserIdAndRequestState(au.getId(), FileAccessRequest.RequestState.CREATED);
-                                for(FileAccessRequest far : fileAccessRequests){
-                                    far.setStateGranted();
-                                    fileAccessRequestService.save(far);
-                                }
-                                file.setFileAccessRequests(fileAccessRequests); 
-                                datafileService.save(file);
-                            }       
-                    }
-                }
-
-            }
-
-            if (sendNotification) {
-                for (AuthenticatedUser au : roleAssigneeService.getExplicitUsers(roleAssignee)) {
-                    userNotificationService.sendNotification(au, new Timestamp(new Date().getTime()), UserNotification.Type.GRANTFILEACCESS, dataset.getId());                
-                }
-             }
-        }
-
-        initMaps();
-        */
-    }
-    
     public List<DataverseRole> getAvailableRoles() {
         List<DataverseRole> roles = new LinkedList<>();
         if (dvObject != null && dvObject.getId() != null) {

From d06259f00b04965571cd4965d34fccd9f0067249 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 20 Sep 2023 14:10:41 -0400
Subject: [PATCH 425/922] Revert "more String->Boolean changes"

This reverts commit 750069b5ac39b0f71f14507c7770f09f36ed3af2.
---
 .../java/edu/harvard/iq/dataverse/DataversePage.java   |  2 +-
 .../edu/harvard/iq/dataverse/DvObjectContainer.java    | 10 ++++++++--
 .../java/edu/harvard/iq/dataverse/SettingsWrapper.java | 10 +++++-----
 .../java/edu/harvard/iq/dataverse/api/Datasets.java    |  7 ++++---
 4 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
index 4ce7042d421..943a74327d5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
@@ -1286,7 +1286,7 @@ public String getCurationLabelSetNameLabel() {
         return setName;
     }
 
-    public Set<Entry<Boolean, String>> getGuestbookEntryOptions() {
+    public Set<Entry<String, String>> getGuestbookEntryOptions() {
         return settingsWrapper.getGuestbookEntryOptions(this.dataverse).entrySet();
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index 7c1e096dc45..da77df786d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -120,9 +120,15 @@ public Boolean getGuestbookEntryAtRequest() {
         return guestbookAtRequest;
     }
 
-    public void setGuestbookEntryAtRequest(Boolean gbAtRequest) {
-        this.guestbookAtRequest = gbAtRequest;
+    public void setGuestbookEntryAtRequest(String gbAtRequest) {
+        if (gbAtRequest != null && gbAtRequest.equals(UNDEFINED_CODE)) {
+            this.guestbookAtRequest = null;
+        } else {
+            //Force to true or false
+            this.guestbookAtRequest = Boolean.valueOf(Boolean.parseBoolean(gbAtRequest));
+        }
     }
+    
 
     /* Dataverse collections can be configured to allow use of Curation labels and have this inheritable value to decide which set of labels to use.
      * This mechanism is similar to that for the storageDriver except that there is an addition option to disable use of labels. 
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index aa5680b4e14..e41a18c1431 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -632,8 +632,8 @@ public String getDefaultMetadataLanguage() {
         }
     }
     
-    public Map<Boolean, String> getGuestbookEntryOptions(DvObjectContainer target) {
-        Map<Boolean, String> currentMap = new HashMap<Boolean, String>();
+    public Map<String, String> getGuestbookEntryOptions(DvObjectContainer target) {
+        Map<String, String> currentMap = new HashMap<String, String>();
         String atDownload = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atdownload");
         String atRequest = BundleUtil.getStringFromBundle("dataverse.guestbookentry.atrequest");
         Optional<Boolean> gbDefault = JvmSettings.GUESTBOOK_AT_REQUEST.lookupOptional(Boolean.class);
@@ -649,9 +649,9 @@ public Map<Boolean, String> getGuestbookEntryOptions(DvObjectContainer target) {
                 useDefault = (defaultOption ? atRequest : atDownload)
                         + BundleUtil.getStringFromBundle("dataverse.inherited");
             }
-            currentMap.put(null, useDefault);
-            currentMap.put(Boolean.TRUE, atRequest);
-            currentMap.put(Boolean.FALSE, atDownload);
+            currentMap.put("null", useDefault);
+            currentMap.put(Boolean.toString(true), atRequest);
+            currentMap.put(Boolean.toString(false), atDownload);
         } else {
             // Setting not defined - leave empty
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 07ac287ddb3..a1191a6f4e6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3924,9 +3924,10 @@ public Response setguestbookEntryAtRequest(@Context ContainerRequestContext crc,
         if (!gbAtRequestOpt.isPresent()) {
             return error(Response.Status.FORBIDDEN, "Guestbook Entry At Request cannot be set. This server is not configured to allow it.");
         }
-        dataset.setGuestbookEntryAtRequest(gbAtRequest);
+        String choice = Boolean.valueOf(gbAtRequest).toString();
+        dataset.setGuestbookEntryAtRequest(choice);
         datasetService.merge(dataset);
-        return ok("Guestbook Entry At Request set to: " + gbAtRequest);
+        return ok("Guestbook Entry At Request set to: " + choice);
     }
     
     @DELETE
@@ -3954,7 +3955,7 @@ public Response resetGuestbookEntryAtRequest(@Context ContainerRequestContext cr
             return error(Response.Status.NOT_FOUND, "No such dataset");
         }
         
-        dataset.setGuestbookEntryAtRequest(null);
+        dataset.setGuestbookEntryAtRequest(DvObjectContainer.UNDEFINED_CODE);
         datasetService.merge(dataset);
         return ok("Guestbook Entry At Request reset to default: " + dataset.getEffectiveGuestbookEntryAtRequest());
     }

From 3ad53d7c0b73260122ec5deecf072edc456d2a8e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 20 Sep 2023 14:54:43 -0400
Subject: [PATCH 426/922] fix Dataverse UI ability to set GB at request param

---
 .../java/edu/harvard/iq/dataverse/DvObjectContainer.java | 9 ++++++---
 .../java/edu/harvard/iq/dataverse/SettingsWrapper.java   | 2 +-
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index da77df786d8..f7d361d76f5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -116,12 +116,15 @@ public boolean getEffectiveGuestbookEntryAtRequest() {
         return gbAtRequest;
     }
     
-    public Boolean getGuestbookEntryAtRequest() {
-        return guestbookAtRequest;
+    public String getGuestbookEntryAtRequest() {
+        if(guestbookAtRequest==null) {
+            return UNDEFINED_CODE;
+        }
+        return Boolean.valueOf(guestbookAtRequest).toString();
     }
 
     public void setGuestbookEntryAtRequest(String gbAtRequest) {
-        if (gbAtRequest != null && gbAtRequest.equals(UNDEFINED_CODE)) {
+        if (gbAtRequest == null || gbAtRequest.equals(UNDEFINED_CODE)) {
             this.guestbookAtRequest = null;
         } else {
             //Force to true or false
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index e41a18c1431..0a1d0effc03 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -649,7 +649,7 @@ public Map<String, String> getGuestbookEntryOptions(DvObjectContainer target) {
                 useDefault = (defaultOption ? atRequest : atDownload)
                         + BundleUtil.getStringFromBundle("dataverse.inherited");
             }
-            currentMap.put("null", useDefault);
+            currentMap.put(DvObjectContainer.UNDEFINED_CODE, useDefault);
             currentMap.put(Boolean.toString(true), atRequest);
             currentMap.put(Boolean.toString(false), atDownload);
         } else {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a1191a6f4e6..55e329bea68 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3888,8 +3888,8 @@ public Response getGuestbookEntryOption(@Context ContainerRequestContext crc, @P
         } catch (WrappedResponse ex) {
             return error(Response.Status.NOT_FOUND, "No such dataset");
         }
-        Boolean gbAtRequest = dataset.getGuestbookEntryAtRequest();
-        if(gbAtRequest == null) {
+        String gbAtRequest = dataset.getGuestbookEntryAtRequest();
+        if(gbAtRequest == null || gbAtRequest.equals(DvObjectContainer.UNDEFINED_CODE)) {
             return ok("Not set on dataset, using the default: " + dataset.getEffectiveGuestbookEntryAtRequest());
         }
         return ok(dataset.getEffectiveGuestbookEntryAtRequest());

From d9102434f53f78692572cc4ddd1cf4bb195a67df Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 20 Sep 2023 17:22:14 -0400
Subject: [PATCH 427/922] revert #5863

---
 .../harvard/iq/dataverse/FileDownload.java    | 163 ------------------
 .../iq/dataverse/GuestbookResponse.java       |  56 +++---
 .../GuestbookResponseServiceBean.java         |  10 +-
 3 files changed, 34 insertions(+), 195 deletions(-)
 delete mode 100644 src/main/java/edu/harvard/iq/dataverse/FileDownload.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownload.java b/src/main/java/edu/harvard/iq/dataverse/FileDownload.java
deleted file mode 100644
index a79281f71f0..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownload.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
-package edu.harvard.iq.dataverse;
-
-import java.io.Serializable;
-import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
-import jakarta.persistence.Transient;
-import jakarta.persistence.CascadeType;
-import jakarta.persistence.OneToOne;
-import jakarta.persistence.MapsId;
-import jakarta.persistence.FetchType;
-import jakarta.persistence.JoinColumn;
-import java.util.Date;
-
-
-/**
- *
- * @author marina
- */
-@Entity
-public class FileDownload implements Serializable { 
-    
-    @Id
-    private Long id;
-    
-    @OneToOne(fetch = FetchType.LAZY)
-    @MapsId
-    private GuestbookResponse guestbookResponse;
-    
-    @Temporal(value = TemporalType.TIMESTAMP)
-    private Date downloadTimestamp;
-    
-    /*
-    Transient Values carry non-written information 
-    that will assist in the download process
-    - selected file ids is a comma delimited list that contains the file ids for multiple download
-    - fileFormat tells the download api which format a subsettable file should be downloaded as
-    */
-    
-    @Transient
-    private String selectedFileIds;
-    
-    @Transient 
-    private String fileFormat;
-    
-    
-    /**
-     * Possible values for downloadType include "Download", "Subset",
-     * or the displayName of an ExternalTool.
-     *
-     * TODO: Types like "Download" and "Subset" should
-     * be defined once as constants (likely an enum) rather than having these
-     * strings duplicated in various places when setDownloadtype() is called.
-     */
-    private String downloadtype;
-    private String sessionId;
-    
-     public FileDownload(){
-        
-    }
-     
-    public FileDownload(FileDownload source){
-        this.setDownloadTimestamp(source.getDownloadTimestamp());
-        this.setDownloadtype(source.getDownloadtype());
-        this.setFileFormat(source.getFileFormat());
-        this.setGuestbookResponse(source.getGuestbookResponse());
-        this.setSelectedFileIds(source.getSelectedFileIds());
-        this.setSessionId(source.getSessionId());
-    }
-    
-    public String getFileFormat() {
-        return fileFormat;
-    }
-
-    //for download
-    public void setFileFormat(String downloadFormat) {
-        this.fileFormat = downloadFormat;
-    }
-    
-    public String getDownloadtype() {
-        return downloadtype;
-    }
-
-    public void setDownloadtype(String downloadtype) {
-        this.downloadtype = downloadtype;
-    }
-    
-    public String getSessionId() {
-        return sessionId;
-    }
-
-    public void setSessionId(String sessionId) {
-        this.sessionId = sessionId;
-    }
-    
-    public String getSelectedFileIds() {
-        return selectedFileIds;
-    }
-
-    public void setSelectedFileIds(String selectedFileIds) {
-        this.selectedFileIds = selectedFileIds;
-    }
-    
-    public Long getId() {
-        return id;
-    }
-
-    public void setId(Long id) {
-        this.id = id;
-    }
-    
-    public Date getDownloadTimestamp(){
-        return this.downloadTimestamp;
-    }
-    
-    public void setDownloadTimestamp(Date downloadTimestamp){
-        this.downloadTimestamp = downloadTimestamp;
-    }
-    
-    
-    public void setGuestbookResponse(GuestbookResponse gbr){
-        this.guestbookResponse = gbr;
-    }
-
-    public GuestbookResponse getGuestbookResponse(){
-        return this.guestbookResponse;
-    }
-    
-    @Override
-    public int hashCode() {
-        int hash = 0;
-        hash += (id != null ? id.hashCode() : 0);
-        return hash;
-    }
-
-    @Override
-    public boolean equals(Object object) {
-        // TODO: Warning - this method won't work in the case the id fields are not set
-        if (!(object instanceof FileDownload)) {
-            return false;
-        }
-        FileDownload other = (FileDownload) object;
-        if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
-            return false;
-        }
-        return true;
-    }
-
-    @Override
-    public String toString() {
-        return "edu.harvard.iq.dataverse.FileDownload[ id=" + id + " ]";
-    }
-    
-    
-}
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 35c299fa5d5..658413c772d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -67,9 +67,6 @@ public class GuestbookResponse implements Serializable {
     @JoinColumn(nullable=true)
     private AuthenticatedUser authenticatedUser;
 
-    @OneToOne(cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},mappedBy="guestbookResponse",fetch = FetchType.LAZY)
-    private FileDownload fileDownload;
-
     @OneToMany(mappedBy="guestbookResponse",cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST},fetch = FetchType.LAZY)
     //private FileAccessRequest fileAccessRequest;
     private List<FileAccessRequest> fileAccessRequests;
@@ -94,15 +91,35 @@ public class GuestbookResponse implements Serializable {
     @Temporal(value = TemporalType.TIMESTAMP)
     private Date responseTime;
     
+    /**
+     * Possible values for downloadType include "Download", "Subset",
+     * or the displayName of an ExternalTool.
+     *
+     * TODO: Types like "Download" and "Subset" should
+     * be defined once as constants (likely an enum) rather than having these
+     * strings duplicated in various places when setDownloadtype() is called.
+     */
+    private String downloadtype;
+    private String sessionId;
+    
     /*
     Transient Values carry non-written information 
     that will assist in the download process
     - writeResponse is set to false when dataset version is draft.
+    - selected file ids is a comma delimited list that contains the file ids for multiple download
+    - fileFormat tells the download api which format a subsettable file should be downloaded as
+
     */
       
     @Transient 
     private boolean writeResponse = true;
 
+    @Transient
+    private String selectedFileIds;
+    
+    @Transient 
+    private String fileFormat;
+
     /**
      * This transient variable is a place to temporarily retrieve the
      * ExternalTool object from the popup when the popup is required on the
@@ -110,6 +127,7 @@ public class GuestbookResponse implements Serializable {
      */
     @Transient
     private ExternalTool externalTool;
+    
 
     public boolean isWriteResponse() {
         return writeResponse;
@@ -120,19 +138,19 @@ public void setWriteResponse(boolean writeResponse) {
     }
 
     public String getSelectedFileIds(){
-        return this.fileDownload.getSelectedFileIds();
+        return this.getSelectedFileIds();
     }
     
     public void setSelectedFileIds(String selectedFileIds) {
-        this.fileDownload.setSelectedFileIds(selectedFileIds);
+        this.setSelectedFileIds(selectedFileIds);
     }
     
     public String getFileFormat() {
-        return this.fileDownload.getFileFormat();
+        return this.getFileFormat();
     }
 
     public void setFileFormat(String downloadFormat) {
-        this.fileDownload.setFileFormat(downloadFormat);
+        this.setFileFormat(downloadFormat);
     }
     
     public ExternalTool getExternalTool() {
@@ -144,10 +162,6 @@ public void setExternalTool(ExternalTool externalTool) {
     }
 
     public GuestbookResponse(){
-        if(this.getFileDownload() == null){
-            this.fileDownload = new FileDownload();
-            this.fileDownload.setGuestbookResponse(this);
-        }
     }
     
     public GuestbookResponse(GuestbookResponse source){
@@ -160,7 +174,7 @@ public GuestbookResponse(GuestbookResponse source){
         this.setDataset(source.getDataset());
         this.setDatasetVersion(source.getDatasetVersion());
         this.setAuthenticatedUser(source.getAuthenticatedUser());
-   
+        this.setSessionId(source.getSessionId());
         List <CustomQuestionResponse> customQuestionResponses = new ArrayList<>();
         if (!source.getCustomQuestionResponses().isEmpty()){
             for (CustomQuestionResponse customQuestionResponse : source.getCustomQuestionResponses() ){
@@ -173,7 +187,6 @@ public GuestbookResponse(GuestbookResponse source){
         }
         this.setCustomQuestionResponses(customQuestionResponses);
         this.setGuestbook(source.getGuestbook());
-        this.setFileDownload(source.getFileDownload());
     }
     
     
@@ -231,7 +244,6 @@ public Date getResponseTime() {
 
     public void setResponseTime(Date responseTime) {
         this.responseTime = responseTime;
-        this.getFileDownload().setDownloadTimestamp(responseTime);
     }
 
     public String getResponseDate() {
@@ -246,14 +258,6 @@ public void setCustomQuestionResponses(List<CustomQuestionResponse> customQuesti
         this.customQuestionResponses = customQuestionResponses;
     }
     
-    public FileDownload getFileDownload(){
-        return fileDownload;
-    }
-    
-    public void setFileDownload(FileDownload fDownload){
-        this.fileDownload = fDownload;
-    }
-    
     public List<FileAccessRequest> getFileAccessRequests(){
         return fileAccessRequests;
     }
@@ -295,21 +299,21 @@ public void setAuthenticatedUser(AuthenticatedUser authenticatedUser) {
     }
     
     public String getDownloadtype() {
-        return this.fileDownload.getDownloadtype();
+        return this.getDownloadtype();
     }
 
     public void setDownloadtype(String downloadtype) {
-        this.fileDownload.setDownloadtype(downloadtype);
+        this.setDownloadtype(downloadtype);
         
     }
     
     public String getSessionId() {
-        return this.fileDownload.getSessionId();
+        return this.getSessionId();
     }
 
     public void setSessionId(String sessionId) {
         
-        this.fileDownload.setSessionId(sessionId);
+        this.setSessionId(sessionId);
     }
     
     public String toHtmlFormattedResponse() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
index b635a0c03a6..6c84f1cacf1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
@@ -63,15 +63,14 @@ public class GuestbookResponseServiceBean {
                 + " and r.dataset_id = o.id "
                 + " and r.guestbook_id = g.id ";*/
     
-    private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, f.downloadtype,"
+    private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, r.downloadtype,"
                 + " m.label, r.dataFile_id, r.name, r.email, r.institution, r.position,"
                 + " o.protocol, o.authority, o.identifier, d.protocol, d.authority, d.identifier "
-                + "from guestbookresponse r, filedownload f, filemetadata m, dvobject o, guestbook g, dvobject d "
+                + "from guestbookresponse r, filemetadata m, dvobject o, guestbook g, dvobject d "
                 + "where "  
                 + "m.datasetversion_id = (select max(datasetversion_id) from filemetadata where datafile_id =r.datafile_id ) "
                 + " and m.datafile_id = r.datafile_id "
                 + " and d.id = r.datafile_id "
-                + " and r.id = f.guestbookresponse_id "
                 + " and r.dataset_id = o.id "
                 + " and r.guestbook_id = g.id ";
     
@@ -79,14 +78,13 @@ public class GuestbookResponseServiceBean {
     // on the guestbook-results.xhtml page (the info we show on the page is 
     // less detailed than what we let the users download as CSV files, so this 
     // query has fewer fields than the one above). -- L.A.
-    private static final String BASE_QUERY_STRING_FOR_PAGE_DISPLAY = "select  r.id, v.value, r.responsetime, f.downloadtype,  m.label, r.name "
-                + "from guestbookresponse r, filedownload f, datasetfieldvalue v, filemetadata m , dvobject o "
+    private static final String BASE_QUERY_STRING_FOR_PAGE_DISPLAY = "select  r.id, v.value, r.responsetime, r.downloadtype,  m.label, r.name "
+                + "from guestbookresponse r, datasetfieldvalue v, filemetadata m , dvobject o "
                 + "where "  
                 + " v.datasetfield_id = (select id from datasetfield f where datasetfieldtype_id = 1 "
                 + " and datasetversion_id = (select max(id) from datasetversion where dataset_id =r.dataset_id )) "
                 + " and m.datasetversion_id = (select max(datasetversion_id) from filemetadata where datafile_id =r.datafile_id ) "
                 + " and m.datafile_id = r.datafile_id "
-                + " and r.id = f.guestbookresponse_id "
                 + " and r.dataset_id = o.id ";
     
     // And a custom query for retrieving *all* the custom question responses, for 

From 4b5710de04193f3745b3cabd5c6fc751299d3009 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 20 Sep 2023 17:30:06 -0400
Subject: [PATCH 428/922] fix revert, renames, add AccessRequest event type

---
 .../java/edu/harvard/iq/dataverse/DatasetPage.java |  2 +-
 .../harvard/iq/dataverse/FileDownloadHelper.java   |  6 +++---
 .../iq/dataverse/FileDownloadServiceBean.java      |  4 +++-
 .../harvard/iq/dataverse/GuestbookResponse.java    | 14 +++++++-------
 .../iq/dataverse/GuestbookResponseServiceBean.java | 12 ++++++------
 .../engine/command/impl/GetUserTracesCommand.java  |  2 +-
 .../V6.0.0.1__9599-guestbook-at-request.sql        |  8 ++++++++
 7 files changed, 29 insertions(+), 19 deletions(-)
 create mode 100644 src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 5ce6207250d..d920ab87bb4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3246,7 +3246,7 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
         } else {
             guestbookResponse.setFileFormat("");
         }
-        guestbookResponse.setDownloadtype("Download");
+        guestbookResponse.setEventType("Download");
     }
 
     /*helper function to filter the selected files into <selected downloadable>, 
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index 0f93f68623a..f574ed46f34 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -72,7 +72,7 @@ private boolean testResponseLength(String value) {
     // terms/etc. popup.
     public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) {
         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
-        guestbookResponse.setDownloadtype("Download");
+        guestbookResponse.setEventType("Download");
          // Note that this method is only ever called from the file-download-popup -
          // meaning we know for the fact that we DO want to save this
          // guestbookResponse permanently in the database.
@@ -93,7 +93,7 @@ public void writeGuestbookAndOpenSubset(GuestbookResponse guestbookResponse) {
 
              PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
              PrimeFaces.current().executeScript("PF('downloadDataSubsetPopup').show()");
-             guestbookResponse.setDownloadtype("Subset");
+             guestbookResponse.setEventType("Subset");
              fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
 
      }
@@ -158,7 +158,7 @@ public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookRe
       * Writes a guestbook entry for either popup scenario: guestbook or terms.
       */
      public boolean writeGuestbookAndShowPreview(GuestbookResponse guestbookResponse) {
-         guestbookResponse.setDownloadtype("Explore");
+         guestbookResponse.setEventType("Explore");
          fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
          return true;
      }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 0758f053470..a80781b16ff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -201,6 +201,8 @@ public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookRe
             return;
         }
 
+        guestbookResponse.setEventType("AccessRequest");
+
         List <DataFile> selectedDataFiles = new ArrayList<>(); //always make sure it's at least an empty List
 
         if(guestbookResponse.getDataFile() != null ){ //one file 'selected' by 'Request Access' button click
@@ -362,7 +364,7 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter
         String localeCode = session.getLocaleCode();
         ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, localeCode);
         // Persist the name of the tool (i.e. "Data Explorer", etc.)
-        guestbookResponse.setDownloadtype(externalTool.getDisplayName());
+        guestbookResponse.setEventType(externalTool.getDisplayName());
         PrimeFaces.current().executeScript(externalToolHandler.getExploreScript());
         // This is the old logic from TwoRavens, null checks and all.
         if (guestbookResponse != null && guestbookResponse.isWriteResponse()
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 658413c772d..d2d1b672716 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -99,7 +99,7 @@ public class GuestbookResponse implements Serializable {
      * be defined once as constants (likely an enum) rather than having these
      * strings duplicated in various places when setDownloadtype() is called.
      */
-    private String downloadtype;
+    private String eventType;
     private String sessionId;
     
     /*
@@ -298,22 +298,22 @@ public void setAuthenticatedUser(AuthenticatedUser authenticatedUser) {
         this.authenticatedUser = authenticatedUser;
     }
     
-    public String getDownloadtype() {
-        return this.getDownloadtype();
+    public String getEventType() {
+        return this.eventType;
     }
 
-    public void setDownloadtype(String downloadtype) {
-        this.setDownloadtype(downloadtype);
+    public void setEventType(String eventType) {
+        this.eventType = eventType;
         
     }
     
     public String getSessionId() {
-        return this.getSessionId();
+        return this.sessionId;
     }
 
     public void setSessionId(String sessionId) {
         
-        this.setSessionId(sessionId);
+        this.sessionId= sessionId;
     }
     
     public String toHtmlFormattedResponse() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
index 6c84f1cacf1..00e44f2dbf6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
@@ -668,7 +668,7 @@ public GuestbookResponse initGuestbookResponseForFragment(DatasetVersion working
         if (dataset.getGuestbook() != null && !dataset.getGuestbook().getCustomQuestions().isEmpty()) {
             initCustomQuestions(guestbookResponse, dataset);
         }
-        guestbookResponse.setDownloadtype("Download");
+        guestbookResponse.setEventType("Download");
 
         guestbookResponse.setDataset(dataset);
         
@@ -722,9 +722,9 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String
         if (dataset.getGuestbook() != null && !dataset.getGuestbook().getCustomQuestions().isEmpty()) {
             initCustomQuestions(guestbookResponse, dataset);
         }
-        guestbookResponse.setDownloadtype("Download");
+        guestbookResponse.setEventType("Download");
         if(downloadFormat.toLowerCase().equals("subset")){
-            guestbookResponse.setDownloadtype("Subset");
+            guestbookResponse.setEventType("Subset");
         }
         if(downloadFormat.toLowerCase().equals("explore")){
             /**
@@ -740,7 +740,7 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String
              * over in the "explore" method of FileDownloadServiceBean just
              * before the guestbookResponse is written.
              */
-            guestbookResponse.setDownloadtype("Explore");
+            guestbookResponse.setEventType("Explore");
         }
         guestbookResponse.setDataset(dataset);
         
@@ -819,7 +819,7 @@ public GuestbookResponse initDefaultGuestbookResponse(Dataset dataset, DataFile
         guestbookResponse.setDataset(dataset);
         guestbookResponse.setResponseTime(new Date());
         guestbookResponse.setSessionId(session.toString());
-        guestbookResponse.setDownloadtype("Download");
+        guestbookResponse.setEventType("Download");
         setUserDefaultResponses(guestbookResponse, session);
         return guestbookResponse;
     }
@@ -840,7 +840,7 @@ public GuestbookResponse initAPIGuestbookResponse(Dataset dataset, DataFile data
         guestbookResponse.setDataset(dataset);
         guestbookResponse.setResponseTime(new Date());
         guestbookResponse.setSessionId(session.toString());
-        guestbookResponse.setDownloadtype("Download");
+        guestbookResponse.setEventType("Download");
         setUserDefaultResponses(guestbookResponse, session, user);
         return guestbookResponse;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java
index e41d70d9804..df0b5d785e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserTracesCommand.java
@@ -212,7 +212,7 @@ public JsonObjectBuilder execute(CommandContext ctxt) throws CommandException {
                         try {
                             JsonObjectBuilder gbe = Json.createObjectBuilder()
                                     .add("id", guestbookResponse.getId())
-                                    .add("downloadType", guestbookResponse.getDownloadtype())
+                                    .add("eventType", guestbookResponse.getEventType())
                                     .add("filename", guestbookResponse.getDataFile().getCurrentName())
                                     .add("date", guestbookResponse.getResponseDate())
                                     .add("guestbookName", guestbookResponse.getGuestbook().getName());
diff --git a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
new file mode 100644
index 00000000000..63b8c8f531e
--- /dev/null
+++ b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
@@ -0,0 +1,8 @@
+ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS eventtype VARCHAR(255);
+ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS sessionid VARCHAR(255);
+
+UPDATE guestbookresponse g 
+    SET eventtype = (SELECT downloadtype FROM filedownload f where f.guestbookresponse_id = g.id), 
+        sessionid = (SELECT sessionid FROM filedownload f where f.guestbookresponse_id=g.id);
+        
+DROP TABLE filedownload;

From 887d26f2fe5c41ca71f0031a9eae6dbfa13e8559 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 21 Sep 2023 10:11:33 +0100
Subject: [PATCH 429/922] Added: docs for deaccessioning API endpoints

---
 doc/sphinx-guides/source/api/native-api.rst | 42 ++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 90f4ad4e800..f46bd0dd17c 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1020,7 +1020,17 @@ Usage example:
 
 Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
 
-Keep in mind that you can combine all of the above query params depending on the results you are looking for.
+By default, deaccessioned dataset versions are not supported by this endpoint and will be ignored in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a not found error if the version is deaccessioned and you do not enable the option described below.
+
+If you want to consider deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?includeDeaccessioned=true"
+
+.. note:: Keep in mind that you can combine all of the above query params depending on the results you are looking for.
 
 Get File Counts in a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1048,6 +1058,16 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts"
 
+By default, deaccessioned dataset versions are not supported by this endpoint and will be ignored in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a not found error if the version is deaccessioned and you do not enable the option described below.
+
+If you want to consider deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?includeDeaccessioned=true"
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1344,6 +1364,26 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/versions/:draft"
 
+Deaccession Dataset
+~~~~~~~~~~~~~~~~~~~
+
+Given a version of a dataset, updates its status to deaccessioned.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+  export VERSIONID=1.0
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession"
+
 Set Citation Date Field Type for a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 1d661e74f2671405143023e22018c5ca197b9c5c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 21 Sep 2023 10:37:24 +0100
Subject: [PATCH 430/922] Added: release notes for #9852

---
 .../9852-files-api-extension-deaccession.md            | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 doc/release-notes/9852-files-api-extension-deaccession.md

diff --git a/doc/release-notes/9852-files-api-extension-deaccession.md b/doc/release-notes/9852-files-api-extension-deaccession.md
new file mode 100644
index 00000000000..c5f6741932a
--- /dev/null
+++ b/doc/release-notes/9852-files-api-extension-deaccession.md
@@ -0,0 +1,10 @@
+Extended the existing endpoints:
+
+- getVersionFiles (/api/datasets/{id}/versions/{versionId}/files)
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts)
+
+The above endpoints now accept a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files or file counts.
+
+Additionally, a new endpoint has been developed to support version deaccessioning through API (Given a dataset and a version).
+
+- deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession)

From 3c7fa8f0eeb34db7d2ca12d4f7eae8e4e02df1d8 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 21 Sep 2023 11:04:58 +0100
Subject: [PATCH 431/922] Added: friendlyType field to DataFile API json
 payload

---
 doc/release-notes/9852-files-api-extension-deaccession.md       | 2 ++
 .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java    | 1 +
 2 files changed, 3 insertions(+)

diff --git a/doc/release-notes/9852-files-api-extension-deaccession.md b/doc/release-notes/9852-files-api-extension-deaccession.md
index c5f6741932a..55698580e3c 100644
--- a/doc/release-notes/9852-files-api-extension-deaccession.md
+++ b/doc/release-notes/9852-files-api-extension-deaccession.md
@@ -8,3 +8,5 @@ The above endpoints now accept a new boolean optional query parameter "includeDe
 Additionally, a new endpoint has been developed to support version deaccessioning through API (Given a dataset and a version).
 
 - deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession)
+
+Finally, the DataFile API payload has been extended to add the field "friendlyType"
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index e5cd72ff5fc..c4f9e47accf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -665,6 +665,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 .add("pidURL", pidURL)
                 .add("filename", fileName)
                 .add("contentType", df.getContentType())
+                .add("friendlyType", df.getFriendlyType())
                 .add("filesize", df.getFilesize())
                 .add("description", fileMetadata.getDescription())
                 .add("categories", getFileCategories(fileMetadata))

From f848f009516ed8fe229403d4262dc5874831cae1 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 21 Sep 2023 12:25:05 -0400
Subject: [PATCH 432/922] use static final values for fixed event types

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java  | 2 +-
 .../edu/harvard/iq/dataverse/FileDownloadHelper.java     | 6 +++---
 .../harvard/iq/dataverse/FileDownloadServiceBean.java    | 2 +-
 .../java/edu/harvard/iq/dataverse/GuestbookResponse.java | 9 +++++++++
 .../iq/dataverse/GuestbookResponseServiceBean.java       | 8 ++++----
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index d920ab87bb4..f99c10b2b79 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3246,7 +3246,7 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
         } else {
             guestbookResponse.setFileFormat("");
         }
-        guestbookResponse.setEventType("Download");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
     }
 
     /*helper function to filter the selected files into <selected downloadable>, 
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index f574ed46f34..a6ae7223d9d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -72,7 +72,7 @@ private boolean testResponseLength(String value) {
     // terms/etc. popup.
     public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) {
         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
-        guestbookResponse.setEventType("Download");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
          // Note that this method is only ever called from the file-download-popup -
          // meaning we know for the fact that we DO want to save this
          // guestbookResponse permanently in the database.
@@ -93,7 +93,7 @@ public void writeGuestbookAndOpenSubset(GuestbookResponse guestbookResponse) {
 
              PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
              PrimeFaces.current().executeScript("PF('downloadDataSubsetPopup').show()");
-             guestbookResponse.setEventType("Subset");
+             guestbookResponse.setEventType(GuestbookResponse.SUBSET);
              fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
 
      }
@@ -158,7 +158,7 @@ public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookRe
       * Writes a guestbook entry for either popup scenario: guestbook or terms.
       */
      public boolean writeGuestbookAndShowPreview(GuestbookResponse guestbookResponse) {
-         guestbookResponse.setEventType("Explore");
+         guestbookResponse.setEventType(GuestbookResponse.EXPLORE);
          fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
          return true;
      }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index a80781b16ff..ab99b904b73 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -201,7 +201,7 @@ public void writeGuestbookResponseAndRequestAccess(GuestbookResponse guestbookRe
             return;
         }
 
-        guestbookResponse.setEventType("AccessRequest");
+        guestbookResponse.setEventType(GuestbookResponse.ACCESS_REQUEST);
 
         List <DataFile> selectedDataFiles = new ArrayList<>(); //always make sure it's at least an empty List
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index d2d1b672716..b27571cdbf5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -127,8 +127,17 @@ public class GuestbookResponse implements Serializable {
      */
     @Transient
     private ExternalTool externalTool;
+
+    /* Event Types - there are four pre-defined values in use.
+     * The type can also be the name of a previewer/explore tool
+     */
     
+    static final String ACCESS_REQUEST = "AccessRequest";
+    static final String DOWNLOAD = "Download";
+    static final String SUBSET = "Subset";
+    static final String EXPLORE = "Explore";
 
+    
     public boolean isWriteResponse() {
         return writeResponse;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
index 00e44f2dbf6..17bc4bea330 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
@@ -668,7 +668,7 @@ public GuestbookResponse initGuestbookResponseForFragment(DatasetVersion working
         if (dataset.getGuestbook() != null && !dataset.getGuestbook().getCustomQuestions().isEmpty()) {
             initCustomQuestions(guestbookResponse, dataset);
         }
-        guestbookResponse.setEventType("Download");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
 
         guestbookResponse.setDataset(dataset);
         
@@ -722,7 +722,7 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String
         if (dataset.getGuestbook() != null && !dataset.getGuestbook().getCustomQuestions().isEmpty()) {
             initCustomQuestions(guestbookResponse, dataset);
         }
-        guestbookResponse.setEventType("Download");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
         if(downloadFormat.toLowerCase().equals("subset")){
             guestbookResponse.setEventType("Subset");
         }
@@ -819,7 +819,7 @@ public GuestbookResponse initDefaultGuestbookResponse(Dataset dataset, DataFile
         guestbookResponse.setDataset(dataset);
         guestbookResponse.setResponseTime(new Date());
         guestbookResponse.setSessionId(session.toString());
-        guestbookResponse.setEventType("Download");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
         setUserDefaultResponses(guestbookResponse, session);
         return guestbookResponse;
     }
@@ -840,7 +840,7 @@ public GuestbookResponse initAPIGuestbookResponse(Dataset dataset, DataFile data
         guestbookResponse.setDataset(dataset);
         guestbookResponse.setResponseTime(new Date());
         guestbookResponse.setSessionId(session.toString());
-        guestbookResponse.setEventType("Download");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
         setUserDefaultResponses(guestbookResponse, session, user);
         return guestbookResponse;
     }

From cd82c2d7d2769b2c1038df953fa7f53ad4eb6adb Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 21 Sep 2023 14:46:59 -0400
Subject: [PATCH 433/922] update queries, missed 'Subset' etc

---
 .../GuestbookResponseServiceBean.java         | 31 ++++++++++++-------
 .../V6.0.0.1__9599-guestbook-at-request.sql   | 28 +++++++++++++++++
 src/main/webapp/dataset.xhtml                 |  6 ++--
 src/main/webapp/dataverse.xhtml               |  2 +-
 src/main/webapp/file-info-fragment.xhtml      |  2 +-
 src/main/webapp/file.xhtml                    |  4 +--
 6 files changed, 55 insertions(+), 18 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
index 17bc4bea330..4800ffd439f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
@@ -724,7 +724,7 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String
         }
         guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
         if(downloadFormat.toLowerCase().equals("subset")){
-            guestbookResponse.setEventType("Subset");
+            guestbookResponse.setEventType(GuestbookResponse.SUBSET);
         }
         if(downloadFormat.toLowerCase().equals("explore")){
             /**
@@ -740,7 +740,7 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String
              * over in the "explore" method of FileDownloadServiceBean just
              * before the guestbookResponse is written.
              */
-            guestbookResponse.setEventType("Explore");
+            guestbookResponse.setEventType(GuestbookResponse.EXPLORE);
         }
         guestbookResponse.setDataset(dataset);
         
@@ -904,29 +904,36 @@ public void save(GuestbookResponse guestbookResponse) {
         em.persist(guestbookResponse);
     }
     
+    
+    /*
+     * Metrics - download counts from GuestbookResponses: Any GuestbookResponse that
+     * is not of eventtype=='AccessRequest' is considered a download. This includes
+     * actual 'Download's, downloads of 'Subset's, and use by 'Explore' tools and
+     * previewers (where eventtype is the previewer name)
+     */
         
-    public Long getCountGuestbookResponsesByDataFileId(Long dataFileId) {
+    public Long getDownloadCountByDataFileId(Long dataFileId) {
         // datafile id is null, will return 0
-        Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.datafile_id  = " + dataFileId);
+        Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.datafile_id  = " + dataFileId + "and eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"'");
         return (Long) query.getSingleResult();
     }
     
-    public Long getCountGuestbookResponsesByDatasetId(Long datasetId) {
-        return getCountGuestbookResponsesByDatasetId(datasetId, null);
+    public Long getDownloadCountByDatasetId(Long datasetId) {
+        return getDownloadCountByDatasetId(datasetId, null);
     }
     
-    public Long getCountGuestbookResponsesByDatasetId(Long datasetId, LocalDate date) {
+    public Long getDownloadCountByDatasetId(Long datasetId, LocalDate date) {
         // dataset id is null, will return 0        
         Query query;
         if(date != null) {
-            query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.dataset_id  = " + datasetId + " and responsetime < '" + date.toString() + "'");
+            query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.dataset_id  = " + datasetId + " and responsetime < '" + date.toString() + "' and eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"'");
         }else {
-            query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.dataset_id  = " + datasetId);
+            query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o  where o.dataset_id  = " + datasetId+ "and eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"'");
         }
         return (Long) query.getSingleResult();
     }    
 
-    public Long getCountOfAllGuestbookResponses() {
+    public Long getTotalDownloadCount() {
         // dataset id is null, will return 0  
         
         // "SELECT COUNT(*)" is notoriously expensive in PostgresQL for large 
@@ -955,10 +962,12 @@ public Long getCountOfAllGuestbookResponses() {
         } catch (IllegalArgumentException iae) {
             // Don't do anything, we'll fall back to using "SELECT COUNT()"
         }
-        Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o;");
+        Query query = em.createNativeQuery("select count(o.id) from GuestbookResponse  o where eventtype != '" + GuestbookResponse.ACCESS_REQUEST +"';");
         return (Long) query.getSingleResult();
     }
     
+    //End Metrics/download counts
+    
     public List<GuestbookResponse> findByAuthenticatedUserId(AuthenticatedUser user) {
         Query query = em.createNamedQuery("GuestbookResponse.findByAuthenticatedUserId"); 
         query.setParameter("authenticatedUserId", user.getId());
diff --git a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
index 63b8c8f531e..df98047b513 100644
--- a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
+++ b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
@@ -6,3 +6,31 @@ UPDATE guestbookresponse g
         sessionid = (SELECT sessionid FROM filedownload f where f.guestbookresponse_id=g.id);
         
 DROP TABLE filedownload;
+
+-- This creates a function that ESTIMATES the size of the
+-- GuestbookResponse table (for the metrics display), instead
+-- of relying on straight "SELECT COUNT(*) ..."
+-- Significant potential savings for an active installation.
+
+CREATE OR REPLACE FUNCTION estimateGuestBookResponseTableSize()
+RETURNS bigint AS $$
+DECLARE
+  estimatedsize bigint;
+BEGIN
+  SELECT CASE WHEN relpages=0 THEN 0
+              ELSE ((reltuples / relpages)
+               * (pg_relation_size('public.guestbookresponse') / current_setting('block_size')::int))::bigint
+               * (SELECT CASE WHEN ((select count(*) from pg_stats where tablename='guestbookresponse') = 0) THEN 1
+               ELSE 1 - (SELECT (most_common_freqs::text::bigint[])[array_position(most_common_vals::text::text[], 'AccessRequest')]
+                         FROM pg_stats WHERE tablename='guestbookresponse' and attname='eventtype') END)
+         END
+     FROM   pg_class
+     WHERE  oid = 'public.guestbookresponse'::regclass INTO estimatedsize;
+
+     if estimatedsize = 0 then
+     SELECT COUNT(id) FROM guestbookresponse INTO estimatedsize;
+     END if;   
+
+  RETURN estimatedsize;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 47ca447da43..a9f1ff339f0 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -540,7 +540,7 @@
                                                 <!-- Classic downloads -->
                                                 <div class="metrics-count-block" jsf:rendered="#{!settingsWrapper.makeDataCountDisplayEnabled}">
                                                     <h:outputFormat value="{0} #{bundle['metrics.downloads']}">
-                                                        <f:param value="#{guestbookResponseServiceBean.getCountGuestbookResponsesByDatasetId(DatasetPage.dataset.id)}"/>
+                                                        <f:param value="#{guestbookResponseServiceBean.getDownloadCountByDatasetId(DatasetPage.dataset.id)}"/>
                                                     </h:outputFormat>
                                                     <span class="glyphicon glyphicon-question-sign tooltip-icon"
                                                           data-toggle="tooltip" data-placement="auto top" data-original-title="#{bundle['metrics.dataset.downloads.default.tip']}"></span>
@@ -562,9 +562,9 @@
                                                           data-toggle="tooltip" data-placement="auto top" data-original-title="#{bundle['metrics.dataset.downloads.makedatacount.tip']}"></span>
                                                     <span jsf:rendered="#{settingsWrapper.getMDCStartDate()!=null}">        
                                                         <h:outputFormat value="#{bundle['metrics.downloads.nonMDC']}">
-                                                            <f:param value="#{guestbookResponseServiceBean.getCountGuestbookResponsesByDatasetId(DatasetPage.dataset.id, settingsWrapper.getMDCStartDate())}"/>
+                                                            <f:param value="#{guestbookResponseServiceBean.getDownloadCountByDatasetId(DatasetPage.dataset.id, settingsWrapper.getMDCStartDate())}"/>
                                                         </h:outputFormat>
-                                                        <span jsf:rendered="#{guestbookResponseServiceBean.getCountGuestbookResponsesByDatasetId(DatasetPage.dataset.id, settingsWrapper.getMDCStartDate()) > 0}" class="glyphicon glyphicon-question-sign tooltip-icon"
+                                                        <span jsf:rendered="#{guestbookResponseServiceBean.getDownloadCountByDatasetId(DatasetPage.dataset.id, settingsWrapper.getMDCStartDate()) > 0}" class="glyphicon glyphicon-question-sign tooltip-icon"
                                                             data-toggle="tooltip" data-placement="auto top" data-original-title="#{bundle['metrics.dataset.downloads.premakedatacount.tip']}"></span>)
                                                         </span>
                                                 </div>
diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml
index e4484d38b97..41e2807c4fd 100644
--- a/src/main/webapp/dataverse.xhtml
+++ b/src/main/webapp/dataverse.xhtml
@@ -458,7 +458,7 @@
                                     </div>
                                     <div id="metrics-content" class="col-xs-8 small text-center">
                                         <h:outputFormat styleClass="metrics-downloads" value="{0} #{bundle['metrics.downloads']}">
-                                            <f:param value="#{guestbookResponseServiceBean.getCountOfAllGuestbookResponses()}"/>
+                                            <f:param value="#{guestbookResponseServiceBean.getTotalDownloadCount()}"/>
                                         </h:outputFormat>
                                     </div>
                                 </div>
diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml
index 33a8d2c3ca5..ca82738f920 100644
--- a/src/main/webapp/file-info-fragment.xhtml
+++ b/src/main/webapp/file-info-fragment.xhtml
@@ -67,7 +67,7 @@
                 </div>
                 <div class="downloads-block" jsf:rendered="#{!editDatafilesPage and !(settingsWrapper.rsyncOnly) and fileMetadata.dataFile.released}">
                     <h:outputFormat styleClass="visible-xs-block visible-sm-block visible-md-inline visible-lg-inline" id="fileDownloadCount" value="{0} #{bundle['metrics.downloads']}">
-                        <f:param value="#{guestbookResponseServiceBean.getCountGuestbookResponsesByDataFileId(fileMetadata.dataFile.id)}"/>
+                        <f:param value="#{guestbookResponseServiceBean.getDownloadCountByDataFileId(fileMetadata.dataFile.id)}"/>
                     </h:outputFormat>
                 </div>
                 <!-- CHECKSUM -->
diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index 3772b3c4dbe..744b593dbd8 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -298,7 +298,7 @@
                                             <!-- Classic downloads -->
                                             <div class="metrics-count-block" jsf:rendered="#{!settingsWrapper.makeDataCountDisplayEnabled}">
                                                 <h:outputFormat value="{0} #{bundle['metrics.downloads']}">
-                                                    <f:param value="#{guestbookResponseServiceBean.getCountGuestbookResponsesByDataFileId(FilePage.fileId)}"/>
+                                                    <f:param value="#{guestbookResponseServiceBean.getDownloadCountByDataFileId(FilePage.fileId)}"/>
                                                 </h:outputFormat>
                                                 <span class="glyphicon glyphicon-question-sign tooltip-icon"
                                                       data-toggle="tooltip" data-placement="auto top" data-original-title="#{bundle['metrics.file.downloads.tip']}"></span>
@@ -306,7 +306,7 @@
                                             <!-- Make Data Count downloads -->
                                             <div class="metrics-count-block" jsf:rendered="#{settingsWrapper.makeDataCountDisplayEnabled}">
                                                 <h:outputFormat value="{0} #{bundle['metrics.downloads']}">
-                                                    <f:param value="#{guestbookResponseServiceBean.getCountGuestbookResponsesByDataFileId(FilePage.fileId)}"/>
+                                                    <f:param value="#{guestbookResponseServiceBean.getDownloadCountByDataFileId(FilePage.fileId)}"/>
                                                 </h:outputFormat>
                                                 <span class="glyphicon glyphicon-question-sign tooltip-icon"
                                                         data-toggle="tooltip" data-placement="auto top" data-original-title="#{bundle['metrics.file.downloads.nonmdc.tip']}"></span>

From e506279ad28018ec71e4990e0d99e36c66fe6a26 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 21 Sep 2023 14:51:37 -0400
Subject: [PATCH 434/922] fix recursive calls

---
 .../java/edu/harvard/iq/dataverse/GuestbookResponse.java  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index b27571cdbf5..3fb0d6691bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -147,19 +147,19 @@ public void setWriteResponse(boolean writeResponse) {
     }
 
     public String getSelectedFileIds(){
-        return this.getSelectedFileIds();
+        return this.selectedFileIds;
     }
     
     public void setSelectedFileIds(String selectedFileIds) {
-        this.setSelectedFileIds(selectedFileIds);
+        this.selectedFileIds = selectedFileIds;
     }
     
     public String getFileFormat() {
-        return this.getFileFormat();
+        return this.fileFormat;
     }
 
     public void setFileFormat(String downloadFormat) {
-        this.setFileFormat(downloadFormat);
+        this.fileFormat = downloadFormat;
     }
     
     public ExternalTool getExternalTool() {

From bfce43eca504192e553d85bce103f88aa481af88 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 21 Sep 2023 15:55:12 -0400
Subject: [PATCH 435/922] update stored function, make update idempotent

---
 .../V6.0.0.1__9599-guestbook-at-request.sql   | 29 ++++++++++++-------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
index df98047b513..fd892fd3356 100644
--- a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
+++ b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
@@ -1,12 +1,19 @@
 ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS eventtype VARCHAR(255);
 ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS sessionid VARCHAR(255);
 
-UPDATE guestbookresponse g 
-    SET eventtype = (SELECT downloadtype FROM filedownload f where f.guestbookresponse_id = g.id), 
-        sessionid = (SELECT sessionid FROM filedownload f where f.guestbookresponse_id=g.id);
-        
-DROP TABLE filedownload;
+DO $$
+    BEGIN 
+        IF EXISTS (select 1 from pg_class where relname='filedownload') THEN 
 
+            UPDATE guestbookresponse g 
+                SET eventtype = (SELECT downloadtype FROM filedownload f where f.guestbookresponse_id = g.id), 
+                    sessionid = (SELECT sessionid FROM filedownload f where f.guestbookresponse_id=g.id);
+            DROP TABLE filedownload;
+        END IF;
+    END
+   $$ ;
+   
+   
 -- This creates a function that ESTIMATES the size of the
 -- GuestbookResponse table (for the metrics display), instead
 -- of relying on straight "SELECT COUNT(*) ..."
@@ -17,18 +24,20 @@ RETURNS bigint AS $$
 DECLARE
   estimatedsize bigint;
 BEGIN
-  SELECT CASE WHEN relpages=0 THEN 0
+  SELECT CASE WHEN relpages<10 THEN 0
               ELSE ((reltuples / relpages)
                * (pg_relation_size('public.guestbookresponse') / current_setting('block_size')::int))::bigint
-               * (SELECT CASE WHEN ((select count(*) from pg_stats where tablename='guestbookresponse') = 0) THEN 1
-               ELSE 1 - (SELECT (most_common_freqs::text::bigint[])[array_position(most_common_vals::text::text[], 'AccessRequest')]
-                         FROM pg_stats WHERE tablename='guestbookresponse' and attname='eventtype') END)
+               * (SELECT CASE WHEN ((select count(*) from pg_stats where tablename='guestbookresponse') = 0 
+                   OR (select array_position(most_common_vals::text::text[], 'AccessRequest') 
+                       FROM pg_stats WHERE tablename='guestbookresponse' AND attname='eventtype') IS NULL) THEN 1
+                   ELSE 1 - (SELECT (most_common_freqs::text::text[])[array_position(most_common_vals::text::text[], 'AccessRequest')]::bigint
+                       FROM pg_stats WHERE tablename='guestbookresponse' and attname='eventtype') END)
          END
      FROM   pg_class
      WHERE  oid = 'public.guestbookresponse'::regclass INTO estimatedsize;
 
      if estimatedsize = 0 then
-     SELECT COUNT(id) FROM guestbookresponse INTO estimatedsize;
+     SELECT COUNT(id) FROM guestbookresponse WHERE eventtype!= 'AccessRequest' INTO estimatedsize;
      END if;   
 
   RETURN estimatedsize;

From efa3d518a8184bef56b59ead26afba3af53ccb03 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 22 Sep 2023 09:56:28 -0400
Subject: [PATCH 436/922] remove todo

---
 .../iq/dataverse/GuestbookResponse.java       | 29 +++++++------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 3fb0d6691bf..203be69cf14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -90,18 +90,19 @@ public class GuestbookResponse implements Serializable {
     
     @Temporal(value = TemporalType.TIMESTAMP)
     private Date responseTime;
-    
-    /**
-     * Possible values for downloadType include "Download", "Subset",
-     * or the displayName of an ExternalTool.
-     *
-     * TODO: Types like "Download" and "Subset" should
-     * be defined once as constants (likely an enum) rather than having these
-     * strings duplicated in various places when setDownloadtype() is called.
-     */
-    private String eventType;
+
     private String sessionId;
+    private String eventType;
+
+    /** Event Types - there are four pre-defined values in use.
+     * The type can also be the name of a previewer/explore tool
+     */
     
+    static final String ACCESS_REQUEST = "AccessRequest";
+    static final String DOWNLOAD = "Download";
+    static final String SUBSET = "Subset";
+    static final String EXPLORE = "Explore";
+
     /*
     Transient Values carry non-written information 
     that will assist in the download process
@@ -128,14 +129,6 @@ public class GuestbookResponse implements Serializable {
     @Transient
     private ExternalTool externalTool;
 
-    /* Event Types - there are four pre-defined values in use.
-     * The type can also be the name of a previewer/explore tool
-     */
-    
-    static final String ACCESS_REQUEST = "AccessRequest";
-    static final String DOWNLOAD = "Download";
-    static final String SUBSET = "Subset";
-    static final String EXPLORE = "Explore";
 
     
     public boolean isWriteResponse() {

From 0350f99697d7099f7a4e89cc41064a3bae25d60b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 22 Sep 2023 10:14:30 -0400
Subject: [PATCH 437/922] add notes, reference issue/PR about the estimation
 method

---
 .../db/migration/V6.0.0.1__9599-guestbook-at-request.sql     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
index fd892fd3356..109963d8382 100644
--- a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
+++ b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
@@ -17,7 +17,12 @@ DO $$
 -- This creates a function that ESTIMATES the size of the
 -- GuestbookResponse table (for the metrics display), instead
 -- of relying on straight "SELECT COUNT(*) ..."
+-- It uses statistics to estimate the number of guestbook entries
+-- and the fraction of them related to downloads,
+-- i.e. those that weren't created for 'AccessRequest' events.
 -- Significant potential savings for an active installation.
+-- See https://github.com/IQSS/dataverse/issues/8840 and 
+-- https://github.com/IQSS/dataverse/pull/8972 for more details
 
 CREATE OR REPLACE FUNCTION estimateGuestBookResponseTableSize()
 RETURNS bigint AS $$

From bbed57df0f2624f6a9300af509906e902541c82b Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Fri, 22 Sep 2023 10:26:02 -0400
Subject: [PATCH 438/922] #9944 add python-dvuploader to client libraries page

---
 doc/sphinx-guides/source/api/client-libraries.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index a25efe3a5f8..d53b9b2d776 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -52,6 +52,8 @@ There are multiple Python modules for interacting with Dataverse APIs.
 
 `EasyDataverse <https://github.com/gdcc/easyDataverse>`_ is a Python library designed to simplify the management of Dataverse datasets in an object-oriented way, giving users the ability to upload, download, and update datasets with ease. By utilizing metadata block configurations, EasyDataverse automatically generates Python objects that contain all the necessary details required to create the native Dataverse JSON format used to create or edit datasets. Adding files and directories is also possible with EasyDataverse and requires no additional API calls. This library is particularly well-suited for client applications such as workflows and scripts as it minimizes technical complexities and facilitates swift development.
 
+`python-dvuploader <https://github.com/JR-1991/python-dvuploader>`_ implements Jim Myers' excellent `dv-uploader <https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader>`_ as a Python module. It offers parallel direct uploads to Dataverse backend storage, streams files directly instead of buffering them in memory, and supports multi-part uploads, chunking data accordingly.
+
 `pyDataverse <https://github.com/gdcc/pyDataverse>`_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger <http://stefankasberger.at>`_ from `AUSSDA - The Austrian Social Science Data Archive <https://aussda.at>`_.  
 
 `UBC's Dataverse Utilities <https://ubc-library-rc.github.io/dataverse_utils/>`_ are a set of Python console utilities which allow one to upload datasets from a tab-separated-value spreadsheet, bulk release multiple datasets, bulk delete unpublished datasets, quickly duplicate records. replace licenses, and more. For additional information see their `PyPi page <https://pypi.org/project/dataverse-utils/>`_.

From b3808c5f5d4b058a5be359c27a35254936a37266 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Fri, 22 Sep 2023 12:20:31 -0400
Subject: [PATCH 439/922] We talked about this on the Containerization Working
 Group meeting on 2023-09-21. Deleting the duplicated chmod and moving the
 bootstrap script copy  to be executed before the chmod.

---
 modules/container-configbaker/Dockerfile | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile
index 44f3806a591..2975b043213 100644
--- a/modules/container-configbaker/Dockerfile
+++ b/modules/container-configbaker/Dockerfile
@@ -26,8 +26,12 @@ RUN true && \
   # Make our working directories
   mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE}
 
-# Get in the scripts and make them executable (just in case...)
+# Get in the scripts 
 COPY maven/scripts maven/solr/update-fields.sh ${SCRIPT_DIR}/
+# Copy the data from scripts/api that provide the common base setup you'd get from the installer.
+# ".dockerignore" will take care of taking only the bare necessities
+COPY maven/setup ${SCRIPT_DIR}/bootstrap/base/
+# Make the scripts executable
 RUN chmod +x ${SCRIPT_DIR}/*.sh ${BOOTSTRAP_DIR}/*/*.sh
 
 # Copy the Solr config bits
@@ -35,10 +39,8 @@ COPY --from=solr /opt/solr/server/solr/configsets/_default ${SOLR_TEMPLATE}/
 COPY maven/solr/*.xml ${SOLR_TEMPLATE}/conf/
 RUN rm ${SOLR_TEMPLATE}/conf/managed-schema.xml
 
-# Copy the data from scripts/api that provide the common base setup you'd get from the installer.
-# ".dockerignore" will take care of taking only the bare necessities
-COPY maven/setup ${SCRIPT_DIR}/bootstrap/base/
-RUN chmod +x ${BOOTSTRAP_DIR}/*/*.sh
+
+
 
 # Set the entrypoint to tini (as a process supervisor)
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

From c82c47e0c08d9a0d507ee980fec78fd7345d4f4a Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Fri, 22 Sep 2023 13:33:57 -0400
Subject: [PATCH 440/922] #9944 python-dvuploader moved to gdcc org

---
 doc/sphinx-guides/source/api/client-libraries.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index d53b9b2d776..4aa5b935e27 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -52,7 +52,7 @@ There are multiple Python modules for interacting with Dataverse APIs.
 
 `EasyDataverse <https://github.com/gdcc/easyDataverse>`_ is a Python library designed to simplify the management of Dataverse datasets in an object-oriented way, giving users the ability to upload, download, and update datasets with ease. By utilizing metadata block configurations, EasyDataverse automatically generates Python objects that contain all the necessary details required to create the native Dataverse JSON format used to create or edit datasets. Adding files and directories is also possible with EasyDataverse and requires no additional API calls. This library is particularly well-suited for client applications such as workflows and scripts as it minimizes technical complexities and facilitates swift development.
 
-`python-dvuploader <https://github.com/JR-1991/python-dvuploader>`_ implements Jim Myers' excellent `dv-uploader <https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader>`_ as a Python module. It offers parallel direct uploads to Dataverse backend storage, streams files directly instead of buffering them in memory, and supports multi-part uploads, chunking data accordingly.
+`python-dvuploader <https://github.com/gdcc/python-dvuploader>`_ implements Jim Myers' excellent `dv-uploader <https://github.com/GlobalDataverseCommunityConsortium/dataverse-uploader>`_ as a Python module. It offers parallel direct uploads to Dataverse backend storage, streams files directly instead of buffering them in memory, and supports multi-part uploads, chunking data accordingly.
 
 `pyDataverse <https://github.com/gdcc/pyDataverse>`_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger <http://stefankasberger.at>`_ from `AUSSDA - The Austrian Social Science Data Archive <https://aussda.at>`_.  
 

From 53a901c8ce60edd4cfd87a179092286dd9520a86 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 22 Sep 2023 14:24:29 -0400
Subject: [PATCH 441/922] update metrics api queries

---
 .../java/edu/harvard/iq/dataverse/GuestbookResponse.java  | 3 +--
 .../harvard/iq/dataverse/metrics/MetricsServiceBean.java  | 8 ++++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 203be69cf14..976f1e084ac 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -98,7 +98,7 @@ public class GuestbookResponse implements Serializable {
      * The type can also be the name of a previewer/explore tool
      */
     
-    static final String ACCESS_REQUEST = "AccessRequest";
+    public static final String ACCESS_REQUEST = "AccessRequest";
     static final String DOWNLOAD = "Download";
     static final String SUBSET = "Subset";
     static final String EXPLORE = "Explore";
@@ -129,7 +129,6 @@ public class GuestbookResponse implements Serializable {
     @Transient
     private ExternalTool externalTool;
 
-
     
     public boolean isWriteResponse() {
         return writeResponse;
diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index 065b42e5afe..837cd518817 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -2,6 +2,7 @@
 
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.GuestbookResponse;
 import edu.harvard.iq.dataverse.Metric;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountUtil.MetricType;
 
@@ -424,6 +425,7 @@ public JsonArray downloadsTimeSeries(Dataverse d) {
                 + "select  distinct COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "') as date, count(id)\n"
                 + "from guestbookresponse\n"
                 + ((d == null) ? "" : "where dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")")
+                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + " group by COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "') order by  COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "');");
 
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -456,6 +458,7 @@ public long downloadsToMonth(String yyyymm, Dataverse d) throws ParseException {
                         + "from guestbookresponse\n"
                         + "where (date_trunc('month', responsetime) <=  to_date('" + yyyymm + "','YYYY-MM')"
                         + "or responsetime is NULL)\n" // includes historic guestbook records without date
+                        + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                     + ((d==null) ? ";": "AND dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ");") 
                 );
                 logger.log(Level.FINE, "Metric query: {0}", query);
@@ -477,6 +480,7 @@ public long downloadsPastDays(int days, Dataverse d) {
                 + "select count(id)\n"
                 + "from guestbookresponse\n"
                 + "where responsetime > current_date - interval '" + days + "' day\n"
+                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + ((d==null) ? ";": "AND dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ");")
         );
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -489,6 +493,7 @@ public JsonArray fileDownloadsTimeSeries(Dataverse d, boolean uniqueCounts) {
                 + " FROM guestbookresponse gb, DvObject ob"
                 + " where ob.id = gb.datafile_id "
                 + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n")
+                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + "group by gb.datafile_id, ob.id, ob.protocol, ob.authority, ob.identifier, to_char(gb.responsetime, 'YYYY-MM') order by to_char(gb.responsetime, 'YYYY-MM');");
 
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -503,6 +508,7 @@ public JsonArray fileDownloads(String yyyymm, Dataverse d, boolean uniqueCounts)
                 + " where ob.id = gb.datafile_id "
                 + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n")
                 + " and date_trunc('month', gb.responsetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
+                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + "group by gb.datafile_id, ob.id, ob.protocol, ob.authority, ob.identifier order by count desc;");
 
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -529,6 +535,7 @@ public JsonArray uniqueDownloadsTimeSeries(Dataverse d) {
                 + " FROM guestbookresponse gb, DvObject ob"
                 + " where ob.id = gb.dataset_id "
                 + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
+                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + "group by gb.dataset_id, ob.protocol, ob.authority, ob.identifier, to_char(gb.responsetime, 'YYYY-MM') order by to_char(gb.responsetime, 'YYYY-MM');");
 
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -546,6 +553,7 @@ public JsonArray uniqueDatasetDownloads(String yyyymm, Dataverse d) {
                 + " where ob.id = gb.dataset_id "
                 + ((d == null) ? "" : " and ob.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                 + " and date_trunc('month', responsetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
+                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + "group by gb.dataset_id, ob.protocol, ob.authority, ob.identifier order by count(distinct email) desc;");
         JsonArrayBuilder jab = Json.createArrayBuilder();
         try {

From 740f63bd852abdd19c7f7aa43b2f641519ae0a66 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 22 Sep 2023 14:52:23 -0400
Subject: [PATCH 442/922] fix query

---
 .../edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index 837cd518817..79369207963 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -425,7 +425,7 @@ public JsonArray downloadsTimeSeries(Dataverse d) {
                 + "select  distinct COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "') as date, count(id)\n"
                 + "from guestbookresponse\n"
                 + ((d == null) ? "" : "where dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")")
-                + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
+                + ((d == null) ? "where ":" and ") + "eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
                 + " group by COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "') order by  COALESCE(to_char(responsetime, 'YYYY-MM'),'" + earliest + "');");
 
         logger.log(Level.FINE, "Metric query: {0}", query);

From 21c4a7fbaa48ace2ba588a72103af619cd70df02 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 25 Sep 2023 10:36:12 -0400
Subject: [PATCH 443/922] downloadtype -> eventtype

---
 .../iq/dataverse/GuestbookResponseServiceBean.java        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
index 4800ffd439f..b0cc41eb448 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponseServiceBean.java
@@ -63,7 +63,7 @@ public class GuestbookResponseServiceBean {
                 + " and r.dataset_id = o.id "
                 + " and r.guestbook_id = g.id ";*/
     
-    private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, r.downloadtype,"
+    private static final String BASE_QUERY_STRING_FOR_DOWNLOAD_AS_CSV = "select r.id, g.name, o.id, r.responsetime, r.eventtype,"
                 + " m.label, r.dataFile_id, r.name, r.email, r.institution, r.position,"
                 + " o.protocol, o.authority, o.identifier, d.protocol, d.authority, d.identifier "
                 + "from guestbookresponse r, filemetadata m, dvobject o, guestbook g, dvobject d "
@@ -78,7 +78,7 @@ public class GuestbookResponseServiceBean {
     // on the guestbook-results.xhtml page (the info we show on the page is 
     // less detailed than what we let the users download as CSV files, so this 
     // query has fewer fields than the one above). -- L.A.
-    private static final String BASE_QUERY_STRING_FOR_PAGE_DISPLAY = "select  r.id, v.value, r.responsetime, r.downloadtype,  m.label, r.name "
+    private static final String BASE_QUERY_STRING_FOR_PAGE_DISPLAY = "select  r.id, v.value, r.responsetime, r.eventtype,  m.label, r.name "
                 + "from guestbookresponse r, datasetfieldvalue v, filemetadata m , dvobject o "
                 + "where "  
                 + " v.datasetfield_id = (select id from datasetfield f where datasetfieldtype_id = 1 "
@@ -735,8 +735,8 @@ public GuestbookResponse initGuestbookResponse(FileMetadata fileMetadata, String
              * "externalTool" for all external tools, including TwoRavens. When
              * clicking "Explore" and then the name of the tool, we want the
              * name of the exploration tool (i.e. "Data Explorer",
-             * etc.) to be persisted as the downloadType. We execute
-             * guestbookResponse.setDownloadtype(externalTool.getDisplayName())
+             * etc.) to be persisted as the eventType. We execute
+             * guestbookResponse.setEventType(externalTool.getDisplayName())
              * over in the "explore" method of FileDownloadServiceBean just
              * before the guestbookResponse is written.
              */

From e5575135b474eca7457a620646d95c4cc839aad9 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 25 Sep 2023 12:36:52 -0400
Subject: [PATCH 444/922] fix popup logic for previews, fix old bug

guestbookAndTermsPopupRequired (and downloadPopupRequired before it) was
not defined and I think therefore was always false, meaning the option
to show a popup never occurred. This may have been OK in practice since
one would have to have accepted the terms popup to show the preview and
preview pane with these buttons. The fix here should show the terms
popup prior to allowing the explore button to be clicked if/when these
buttons ever show and one hasn't already accepted the terms (and
termsMet is therefore true).
---
 src/main/webapp/file.xhtml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml
index 744b593dbd8..f69b5c35afd 100644
--- a/src/main/webapp/file.xhtml
+++ b/src/main/webapp/file.xhtml
@@ -364,12 +364,12 @@
                                         <ui:param name="guestbookResponseService" value="#{FilePage.guestbookResponseService}"/>
                                         <ui:param name="fileDownloadService" value="#{FilePage.fileDownloadService}"/>
                                         <ui:param name="lockedFromDownload" value="#{FilePage.lockedFromDownload}"/>
-                                        <ui:param name="termsGuestbookPopupAction" value="#{FilePage.termsGuestbookPopupAction}"/>
+                                        <ui:param name="termsGuestbookPopupAction" value="#{bundle.download}"/>
                                         <ui:param name="guestbookPopupRequiredAtDownload" value="#{FilePage.guestbookPopupRequiredAtDownload}"/>
                                     </ui:include>
                                 </ui:fragment>
                                 <!-- PREVIEW EXTERNAL TOOL -->
-                                <ui:fragment rendered="#{(!FilePage.guestbookAndTermsPopupRequired) or (FilePage.guestbookAndTermsPopupRequired and FilePage.termsMet)}">
+                                <ui:fragment rendered="#{(!FilePage.guestbookAndTermsPopupRequired) or FilePage.termsMet}">
                                 <div class="btn-toolbar margin-bottom" role="toolbar" aria-label="#{bundle['file.previewTab.button.label']}">
                                     <!-- Preview Button Group -->
                                     <div class="btn-group" jsf:rendered="#{FilePage.allAvailableTools.size() > 1 and fileDownloadHelper.canDownloadFile(FilePage.fileMetadata)}">
@@ -392,7 +392,9 @@
                                     <!-- END: Preview Button Group -->
                                     <div class="btn-group" jsf:rendered="#{(FilePage.toolsWithPreviews.size() > 0 or FilePage.queryTools.size() > 0) and fileDownloadHelper.canDownloadFile(FilePage.fileMetadata)}">
                                         <!-- Modular/Configured Explore Tool -->
-                                        <h:commandLink rendered="#{!guestbookAndTermsPopupRequired}" 
+                                        <ui:remove><!-- Not sure this div will ever be rendered when terms are required and FilePage.termsMet isn't true. Leaving this logic in in case that is incorrect or changes going forward --></ui:remove>
+                            
+                                        <h:commandLink rendered="#{(!FilePage.guestbookAndTermsPopupRequired) or FilePage.termsMet}" 
                                                          type="submit"
                                                          styleClass="btn btn-default #{(FilePage.fileMetadata.dataFile.ingestInProgress) ? 'disabled' : ''}"
                                                          disabled="#{(FilePage.fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
@@ -403,7 +405,7 @@
                                                 <f:param value="#{FilePage.selectedTool.getDisplayNameLang()}"/>
                                             </h:outputFormat>
                                         </h:commandLink>
-                                        <p:commandLink rendered="#{guestbookAndTermsPopupRequired}"
+                                        <p:commandLink rendered="#{FilePage.guestbookAndTermsPopupRequired and !FilePage.termsMet}"
                                                          action="#{FilePage.guestbookResponseService.modifyDatafileAndFormat(FilePage.guestbookResponse, FilePage.fileMetadata, 'externalTool', FilePage.selectedTool)}"
                                                          styleClass="btn btn-default"
                                                          disabled="#{(FilePage.fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}"
@@ -411,6 +413,7 @@
                                                          process="@this"
                                                          update="@widgetVar(guestbookAndTermsPopup)"
                                                          oncomplete="PF('guestbookAndTermsPopup').show();handleResizeDialog('guestbookAndTermsPopup');">
+                                            <f:actionListener binding="#{FilePage.setTermsGuestbookPopupAction(bundle.download)}"/>
                                             <span class="glyphicon glyphicon-#{FilePage.selectedTool.exploreTool ? 'equalizer' : 'new-window'}"></span>
                                             <h:outputFormat value="#{FilePage.selectedTool.exploreTool ? bundle['file.previewTab.exploreBtn'] : bundle['file.previewTab.openBtn']}">
                                                 <f:param value="#{bundle.explore}"/>

From b6e3613a4d3c3dc471aac0a2e9413d126a80afc0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 25 Sep 2023 14:46:13 -0400
Subject: [PATCH 445/922] add popup for dataset configure external tool #9589

---
 .../java/edu/harvard/iq/dataverse/DatasetPage.java     | 10 ++++++++++
 src/main/webapp/dataset.xhtml                          | 10 ++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 78ccb5542ce..0ed33fd216c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -392,6 +392,8 @@ public void setShowIngestSuccess(boolean showIngestSuccess) {
     List<ExternalTool> fileQueryTools = new ArrayList<>();
     private List<ExternalTool> datasetExploreTools;
     private List<ExternalTool> datasetConfigureTools;
+    // The selected dataset-level configure tool
+    private ExternalTool datasetConfigureTool;
 
     public Boolean isHasRsyncScript() {
         return hasRsyncScript;
@@ -5578,6 +5580,14 @@ public List<ExternalTool> getDatasetConfigureTools() {
         return datasetConfigureTools;
     }
 
+    public ExternalTool getDatasetConfigureTool() {
+        return datasetConfigureTool;
+    }
+
+    public void setDatasetConfigureTool(ExternalTool datasetConfigureTool) {
+        this.datasetConfigureTool = datasetConfigureTool;
+    }
+
     Boolean thisLatestReleasedVersion = null;
 
     public boolean isThisLatestReleasedVersion() {
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index b76b33a267c..d7cc18e68a2 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -458,9 +458,9 @@
                                                             <!-- Configure tool links -->
                                                             <ui:repeat var="tool" value="#{DatasetPage.datasetConfigureTools}">
                                                                 <li>
-                                                                    <h:commandLink styleClass="btn-explore" action="#{DatasetPage.explore(tool)}">
+                                                                    <p:commandLink styleClass="btn-explore" actionListener="#{DatasetPage.setDatasetConfigureTool(tool)}" oncomplete="PF('configureToolDialog').show();">
                                                                         <h:outputText value="#{tool.getDisplayNameLang()}"/>
-                                                                    </h:commandLink>
+                                                                    </p:commandLink>
                                                                 </li>
                                                             </ui:repeat>
                                                         </ui:fragment>
@@ -1001,6 +1001,12 @@
                             </button>
                         </div>
                     </p:dialog>
+                    <p:dialog id="configureToolDialog" styleClass="smallPopUp" header="#{DatasetPage.datasetConfigureTool.displayNameLang}" widgetVar="configureToolDialog" modal="true">
+                        <p class="help-block">
+                            <h:outputFormat value="#{DatasetPage.datasetConfigureTool.description}" escape="false"/>
+                            <p:commandLink styleClass="btn-explore" action="#{DatasetPage.explore(DatasetPage.datasetConfigureTool)}" oncomplete="PF('configureToolDialog').hide();"/>
+                        </p>
+                    </p:dialog>
                     <p:dialog id="citationsDialog" styleClass="smallPopUp" header="#{bundle['metrics.citations.dialog.header']}" widgetVar="citationsDialog" modal="true">
                         <p class="help-block">
                             <h:outputFormat value="#{bundle['metrics.citations.dialog.help']}" escape="false">

From cdd3a721deefe659151ab89b2c75cd6f3af016a5 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 25 Sep 2023 18:23:07 -0400
Subject: [PATCH 446/922] 9952 - add missing <> chars for license

---
 .../edu/harvard/iq/dataverse/util/SignpostingResources.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java b/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java
index 2c9b7167059..21abd2d7034 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java
@@ -78,7 +78,7 @@ public String getLinks() {
         type = "<https://schema.org/AboutPage>;rel=\"type\",<" + defaultFileTypeValue + ">;rel=\"type\"";
         valueList.add(type);
 
-        String licenseString = DatasetUtil.getLicenseURI(workingDatasetVersion) + ";rel=\"license\"";
+        String licenseString = "<" + DatasetUtil.getLicenseURI(workingDatasetVersion) + ">;rel=\"license\"";
         valueList.add(licenseString);
 
         String linkset = "<" + systemConfig.getDataverseSiteUrl() + "/api/datasets/:persistentId/versions/"

From 3a4d8f98053ff726c617a45c5ad15d2f3059c138 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 25 Sep 2023 18:25:28 -0400
Subject: [PATCH 447/922] 9953 - don't wrap linkset in a data element

also remove @AuthRequired per #9466
---
 .../harvard/iq/dataverse/api/Datasets.java    | 33 ++++++++++---------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 98bc42f75b0..3b0bc3e0fcf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -580,26 +580,27 @@ public Response getVersionMetadataBlock(@Context ContainerRequestContext crc,
      * @return
      */
     @GET
-    @AuthRequired
     @Path("{id}/versions/{versionId}/linkset")
-    public Response getLinkset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        if ( ":draft".equals(versionId) ) {
+    public Response getLinkset(@PathParam("id") String datasetId, @PathParam("versionId") String versionId,
+            @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        if (":draft".equals(versionId)) {
             return badRequest("Signposting is not supported on the :draft version");
         }
-        User user = getRequestUser(crc);
-        return response(req -> {
+        DataverseRequest req = createDataverseRequest(null);
+        try {
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            return ok(Json.createObjectBuilder().add(
-                    "linkset",
-                    new SignpostingResources(
-                            systemConfig,
-                            dsv,
-                            JvmSettings.SIGNPOSTING_LEVEL1_AUTHOR_LIMIT.lookupOptional().orElse(""),
-                            JvmSettings.SIGNPOSTING_LEVEL1_ITEM_LIMIT.lookupOptional().orElse("")
-                    ).getJsonLinkset()
-            )
-            );
-        }, user);
+            return Response
+                    .ok(Json.createObjectBuilder()
+                            .add("linkset",
+                                    new SignpostingResources(systemConfig, dsv,
+                                            JvmSettings.SIGNPOSTING_LEVEL1_AUTHOR_LIMIT.lookupOptional().orElse(""),
+                                            JvmSettings.SIGNPOSTING_LEVEL1_ITEM_LIMIT.lookupOptional().orElse(""))
+                                                    .getJsonLinkset())
+                            .build())
+                    .type(MediaType.APPLICATION_JSON).build();
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
     }
 
     @GET

From 869d24266bc305d4b008975d8ebe0dd5911063a2 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 25 Sep 2023 18:26:50 -0400
Subject: [PATCH 448/922] add null check to avoid any remaining cases of 9954

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index d20175b6e1a..7cb5bfa3850 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -6144,7 +6144,7 @@ public String getWebloaderUrlForDataset(Dataset d) {
     String signpostingLinkHeader = null;
 
     public String getSignpostingLinkHeader() {
-        if (!workingVersion.isReleased()) {
+        if ((workingVersion==null) || (!workingVersion.isReleased())) {
             return null;
         }
         if (signpostingLinkHeader == null) {

From 2332c1c5b815e737f7e2471d40d31b0fac179c82 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 25 Sep 2023 18:38:05 -0400
Subject: [PATCH 449/922] release note

---
 doc/release-notes/9955-Signposting-updates.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/release-notes/9955-Signposting-updates.md

diff --git a/doc/release-notes/9955-Signposting-updates.md b/doc/release-notes/9955-Signposting-updates.md
new file mode 100644
index 00000000000..bf0c7bc646b
--- /dev/null
+++ b/doc/release-notes/9955-Signposting-updates.md
@@ -0,0 +1 @@
+This release fixes two issues (#9952, #9953) where the Signposting output did not match the Signposting specification.
\ No newline at end of file

From 5978d71337fd1bfbecd42b7bd88b7b4193dbc6ad Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 26 Sep 2023 07:07:00 -0400
Subject: [PATCH 450/922] 9957- use ld+json

---
 doc/release-notes/9955-Signposting-updates.md                 | 2 +-
 doc/sphinx-guides/source/api/native-api.rst                   | 2 +-
 .../edu/harvard/iq/dataverse/DatasetFieldServiceBean.java     | 3 ++-
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java      | 2 +-
 .../edu/harvard/iq/dataverse/util/SignpostingResources.java   | 4 ++--
 5 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/doc/release-notes/9955-Signposting-updates.md b/doc/release-notes/9955-Signposting-updates.md
index bf0c7bc646b..92168231895 100644
--- a/doc/release-notes/9955-Signposting-updates.md
+++ b/doc/release-notes/9955-Signposting-updates.md
@@ -1 +1 @@
-This release fixes two issues (#9952, #9953) where the Signposting output did not match the Signposting specification.
\ No newline at end of file
+This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification.
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 56d245f97c0..e181a2a5546 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2196,7 +2196,7 @@ Signposting involves the addition of a `Link <https://tools.ietf.org/html/rfc598
 
 Here is an example of a "Link" header:
 
-``Link: <https://doi.org/10.5072/FK2/YD5QDG>;rel="cite-as", <https://doi.org/10.5072/FK2/YD5QDG>;rel="describedby";type="application/vnd.citationstyles.csl+json",<https://demo.dataverse.org/api/datasets/export?exporter=schema.org&persistentId=doi:10.5072/FK2/YD5QDG>;rel="describedby";type="application/json+ld", <https://schema.org/AboutPage>;rel="type",<https://schema.org/Dataset>;rel="type", https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/customlicense?persistentId=doi:10.5072/FK2/YD5QDG;rel="license", <https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/linkset?persistentId=doi:10.5072/FK2/YD5QDG> ; rel="linkset";type="application/linkset+json"``
+``Link: <https://doi.org/10.5072/FK2/YD5QDG>;rel="cite-as", <https://doi.org/10.5072/FK2/YD5QDG>;rel="describedby";type="application/vnd.citationstyles.csl+json",<https://demo.dataverse.org/api/datasets/export?exporter=schema.org&persistentId=doi:10.5072/FK2/YD5QDG>;rel="describedby";type="application/ld+json", <https://schema.org/AboutPage>;rel="type",<https://schema.org/Dataset>;rel="type", <https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/customlicense?persistentId=doi:10.5072/FK2/YD5QDG>;rel="license", <https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/linkset?persistentId=doi:10.5072/FK2/YD5QDG> ; rel="linkset";type="application/linkset+json"``
 
 The URL for linkset information is discoverable under the ``rel="linkset";type="application/linkset+json`` entry in the "Link" header, such as in the example above.
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
index 620d4bf3e09..ce2b00086ec 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java
@@ -500,7 +500,8 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep
                         .setRetryHandler(new DefaultHttpRequestRetryHandler(3, false))
                         .build()) {
                     HttpGet httpGet = new HttpGet(retrievalUri);
-                    httpGet.addHeader("Accept", "application/json+ld, application/json");
+                    //application/json+ld is for backward compatibility
+                    httpGet.addHeader("Accept", "application/ld+json, application/json+ld, application/json");
 
                     HttpResponse response = httpClient.execute(httpGet);
                     String data = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 3b0bc3e0fcf..b9a104d8eaa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -272,7 +272,7 @@ public Response getDataset(@Context ContainerRequestContext crc, @PathParam("id"
     
     @GET
     @Path("/export")
-    @Produces({"application/xml", "application/json", "application/html" })
+    @Produces({"application/xml", "application/json", "application/html", "application/ld+json" })
     public Response exportDataset(@QueryParam("persistentId") String persistentId, @QueryParam("exporter") String exporter, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) {
 
         try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java b/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java
index 21abd2d7034..1826689b892 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SignpostingResources.java
@@ -71,7 +71,7 @@ public String getLinks() {
 
         String describedby = "<" + ds.getGlobalId().asURL().toString() + ">;rel=\"describedby\"" + ";type=\"" + "application/vnd.citationstyles.csl+json\"";
         describedby += ",<" + systemConfig.getDataverseSiteUrl() + "/api/datasets/export?exporter=schema.org&persistentId="
-                + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() + ">;rel=\"describedby\"" + ";type=\"application/json+ld\"";
+                + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier() + ">;rel=\"describedby\"" + ";type=\"application/ld+json\"";
         valueList.add(describedby);
 
         String type = "<https://schema.org/AboutPage>;rel=\"type\"";
@@ -116,7 +116,7 @@ public JsonArrayBuilder getJsonLinkset() {
                         systemConfig.getDataverseSiteUrl() + "/api/datasets/export?exporter=schema.org&persistentId=" + ds.getProtocol() + ":" + ds.getAuthority() + "/" + ds.getIdentifier()
                 ).add(
                         "type",
-                        "application/json+ld"
+                        "application/ld+json"
                 )
         );
         JsonArrayBuilder linksetJsonObj = Json.createArrayBuilder();

From 89dfb012458a93ec563e9c81f9458c2919b23d51 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 26 Sep 2023 10:57:23 -0400
Subject: [PATCH 451/922] #9589 fix update for dialog

---
 src/main/webapp/dataset.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index d7cc18e68a2..5d880e8108d 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -458,7 +458,7 @@
                                                             <!-- Configure tool links -->
                                                             <ui:repeat var="tool" value="#{DatasetPage.datasetConfigureTools}">
                                                                 <li>
-                                                                    <p:commandLink styleClass="btn-explore" actionListener="#{DatasetPage.setDatasetConfigureTool(tool)}" oncomplete="PF('configureToolDialog').show();">
+                                                                    <p:commandLink styleClass="btn-explore" actionListener="#{DatasetPage.setDatasetConfigureTool(tool)}" process="@this" update="datasetForm:configureToolDialog" oncomplete="PF('configureToolDialog').show();">
                                                                         <h:outputText value="#{tool.getDisplayNameLang()}"/>
                                                                     </p:commandLink>
                                                                 </li>

From e3fbd0287392aa6652cf23f32f849c17812a4fd8 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 26 Sep 2023 12:29:01 -0400
Subject: [PATCH 452/922] Update test

---
 src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java
index 17eba4770f1..b41f62ae28f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java
@@ -92,7 +92,7 @@ public void testSignposting() {
 
         String responseString = linksetResponse.getBody().asString();
 
-        JsonObject data = JsonUtil.getJsonObject(responseString).getJsonObject("data");
+        JsonObject data = JsonUtil.getJsonObject(responseString);
         JsonObject lso = data.getJsonArray("linkset").getJsonObject(0);
         System.out.println("Linkset: " + lso.toString());
 

From a8872056358186feff8ae7ba422c27b6449e188a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 26 Sep 2023 13:13:38 -0400
Subject: [PATCH 453/922] Fix merge issue

---
 src/main/java/edu/harvard/iq/dataverse/api/Files.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index ba1eea05fc0..82811162d52 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -840,7 +840,7 @@ public Response getFixityAlgorithm() {
     public Response getFileDownloadCount(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
         return response(req -> {
             DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
-            return ok(guestbookResponseService.getCountGuestbookResponsesByDataFileId(dataFile.getId()).toString());
+            return ok(guestbookResponseService.getDownloadCountByDataFileId(dataFile.getId()).toString());
         }, getRequestUser(crc));
     }
 

From 3dd4564dc56a1132fcda7301a358e8f1f802752b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 26 Sep 2023 18:29:42 +0100
Subject: [PATCH 454/922] Added: ignoreOriginalTabularSize optional query
 parameter to getDownloadSize datasets API endpoint

---
 .../iq/dataverse/DatasetServiceBean.java      | 13 +++--
 .../harvard/iq/dataverse/api/Datasets.java    | 29 ++++++----
 .../impl/GetDatasetStorageSizeCommand.java    | 18 +++---
 .../impl/GetDataverseStorageSizeCommand.java  |  2 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 55 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  7 +++
 6 files changed, 98 insertions(+), 26 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
index 52eb5868c35..4799502a6e3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
@@ -788,13 +788,13 @@ public void exportDataset(Dataset dataset, boolean forceReExport) {
                 }
             }
         }
-        
+
     }
 
     //get a string to add to save success message
     //depends on page (dataset/file) and user privleges
     public String getReminderString(Dataset dataset, boolean canPublishDataset, boolean filePage, boolean isValid) {
-       
+
         String reminderString;
 
         if (canPublishDataset) {
@@ -1015,12 +1015,12 @@ public void obtainPersistentIdentifiersForDatafiles(Dataset dataset) {
     }
 
     public long findStorageSize(Dataset dataset) throws IOException {
-        return findStorageSize(dataset, false, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
+        return findStorageSize(dataset, false, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
     }
 
 
     public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws IOException {
-        return findStorageSize(dataset, countCachedExtras, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
+        return findStorageSize(dataset, countCachedExtras, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
     }
 
     /**
@@ -1028,6 +1028,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
      *
      * @param dataset
      * @param countCachedExtras boolean indicating if the cached disposable extras should also be counted
+     * @param countOriginalTabularSize boolean indicating if the size of the stored original tabular files should also be counted, in addition to the main tab-delimited file size
      * @param mode String indicating whether we are getting the result for storage (entire dataset) or download version based
      * @param version optional param for dataset version
      * @return total size
@@ -1036,7 +1037,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
      * default mode, the method doesn't need to access the storage system, as the
      * sizes of the main files are recorded in the database)
      */
-    public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
+    public long findStorageSize(Dataset dataset, boolean countCachedExtras, boolean countOriginalTabularSize, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
         long total = 0L;
 
         if (dataset.isHarvested()) {
@@ -1062,7 +1063,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatas
                 total += datafile.getFilesize();
 
                 if (!countCachedExtras) {
-                    if (datafile.isTabularData()) {
+                    if (datafile.isTabularData() && countOriginalTabularSize) {
                         // count the size of the stored original, in addition to the main tab-delimited file:
                         Long originalFileSize = datafile.getDataTable().getOriginalFileSize();
                         if (originalFileSize != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 62d87b198fe..a39347ef64e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2947,25 +2947,32 @@ public Response getMakeDataCountMetricCurrentMonth(@PathParam("id") String idSup
         String nullCurrentMonth = null;
         return getMakeDataCountMetric(idSupplied, metricSupplied, nullCurrentMonth, country);
     }
-    
+
     @GET
     @AuthRequired
     @Path("{identifier}/storagesize")
-    public Response getStorageSize(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("includeCached") boolean includeCached,
-                                   @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse {
-
+    public Response getStorageSize(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("includeCached") boolean includeCached) {
         return response(req -> ok(MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.storage"),
-                execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), includeCached, GetDatasetStorageSizeCommand.Mode.STORAGE, null)))), getRequestUser(crc));
+                execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), includeCached, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null)))), getRequestUser(crc));
     }
-    
+
     @GET
     @AuthRequired
     @Path("{identifier}/versions/{versionId}/downloadsize")
-    public Response getDownloadSize(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("versionId") String version,
-                                    @Context UriInfo uriInfo, @Context HttpHeaders headers) throws WrappedResponse {
-
-        return response(req -> ok(MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.download"),
-                execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), false, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers))))), getRequestUser(crc));
+    public Response getDownloadSize(@Context ContainerRequestContext crc,
+                                    @PathParam("identifier") String dvIdtf,
+                                    @PathParam("versionId") String version,
+                                    @QueryParam("ignoreOriginalTabularSize") boolean ignoreOriginalTabularSize,
+                                    @Context UriInfo uriInfo,
+                                    @Context HttpHeaders headers) {
+        return response(req -> {
+            Long datasetStorageSize = execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), false, !ignoreOriginalTabularSize, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers)));
+            String message = MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.download"), datasetStorageSize);
+            JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+            jsonObjectBuilder.add("message", message);
+            jsonObjectBuilder.add("storageSize", datasetStorageSize);
+            return ok(jsonObjectBuilder);
+        }, getRequestUser(crc));
     }
 
     @GET
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
index f1f27fdcee2..eebb8dd9e00 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
@@ -7,7 +7,6 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetVersion;
-import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -15,6 +14,7 @@
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+
 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
@@ -32,47 +32,49 @@ public class GetDatasetStorageSizeCommand extends AbstractCommand<Long> {
 
     private final Dataset dataset;
     private final Boolean countCachedFiles;
+    private final Boolean countOriginalTabularSize;
     private final Mode mode;
     private final DatasetVersion version;
     
     public enum Mode {
 
         STORAGE, DOWNLOAD
-    };
+    }
 
     public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target) {
         super(aRequest, target);
         dataset = target;
         countCachedFiles = false;
+        countOriginalTabularSize = true;
         mode = Mode.DOWNLOAD;
         version = null;
     }
 
-    public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles, Mode mode, DatasetVersion version) {
+    public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles, boolean countOriginalTabularSize, Mode mode, DatasetVersion version) {
         super(aRequest, target);
         dataset = target;
         this.countCachedFiles = countCachedFiles;
+        this.countOriginalTabularSize = countOriginalTabularSize;
         this.mode = mode;
         this.version = version;
     }
 
     @Override
     public Long execute(CommandContext ctxt) throws CommandException {
-        logger.fine("getDataverseStorageSize called on " + dataset.getDisplayName());
-
         if (dataset == null) {
             // should never happen - must indicate some data corruption in the database
             throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.listing.error"), this);
         }
 
+        logger.fine("getDataverseStorageSize called on " + dataset.getDisplayName());
+
         try {
-            return ctxt.datasets().findStorageSize(dataset, countCachedFiles, mode, version);
+            return ctxt.datasets().findStorageSize(dataset, countCachedFiles, countOriginalTabularSize, mode, version);
         } catch (IOException ex) {
             throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.datasize.ioerror"), this);
         }
-
     }
-    
+
     @Override
     public Map<String, Set<Permission>> getRequiredPermissions() {
         // for data file check permission on owning dataset
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java
index 57912a6b4bd..9f93f6747ea 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java
@@ -59,7 +59,7 @@ public Long execute(CommandContext ctxt) throws CommandException {
             }
             
             try {
-                total += ctxt.datasets().findStorageSize(dataset, countCachedFiles, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
+                total += ctxt.datasets().findStorageSize(dataset, countCachedFiles, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
             } catch (IOException ex) {
                 throw new CommandException(BundleUtil.getStringFromBundle("dataverse.datasize.ioerror"), this);
             }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 5c1eb66b63d..929882fe95a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3630,4 +3630,59 @@ public void deaccessionDataset() {
         deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
     }
+
+    @Test
+    public void getDownloadSize() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Creating test text files
+        String testFileName1 = "test_1.txt";
+        String testFileName2 = "test_2.txt";
+
+        int testFileSize1 = 50;
+        int testFileSize2 = 200;
+
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[testFileSize1], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[testFileSize2], apiToken);
+
+        int expectedTextFilesStorageSize = testFileSize1 + testFileSize2;
+
+        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, false, apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data.storageSize", equalTo(expectedTextFilesStorageSize));
+
+        // Upload test tabular file
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get the original tabular file size
+        int tabularOriginalSize = Integer.parseInt(uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.filesize"));
+
+        // Get the size ignoring the original tabular file sizes
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, true, apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        int actualSizeIgnoringOriginalTabularSizes = Integer.parseInt(getDownloadSizeResponse.getBody().jsonPath().getString("data.storageSize"));
+        // Assert that the size has been incremented with the last uploaded file
+        assertTrue(actualSizeIgnoringOriginalTabularSizes > expectedTextFilesStorageSize);
+
+        // Get the size including the original tabular file sizes
+        int expectedSizeIncludingOriginalTabularSizes = tabularOriginalSize + actualSizeIgnoringOriginalTabularSizes;
+
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, false, apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data.storageSize", equalTo(expectedSizeIncludingOriginalTabularSizes));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 8c6a2d6e75d..ecf26bd26ae 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3409,4 +3409,11 @@ static Response deaccessionDataset(Integer datasetId, String version, String api
                 .body(jsonString)
                 .put("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
     }
+
+    static Response getDownloadSize(Integer datasetId, String version, boolean ignoreOriginalTabularSize, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .queryParam("ignoreOriginalTabularSize", ignoreOriginalTabularSize)
+                .get("/api/datasets/" + datasetId + "/versions/" + version + "/downloadsize");
+    }
 }

From 9b4a4827a6173be91e6a8266d4ad12ad890e38b7 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 26 Sep 2023 16:06:30 -0400
Subject: [PATCH 455/922] add links to dataset configure popup #9589

---
 src/main/webapp/dataset.xhtml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 5d880e8108d..7cc6db65b2a 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1004,8 +1004,15 @@
                     <p:dialog id="configureToolDialog" styleClass="smallPopUp" header="#{DatasetPage.datasetConfigureTool.displayNameLang}" widgetVar="configureToolDialog" modal="true">
                         <p class="help-block">
                             <h:outputFormat value="#{DatasetPage.datasetConfigureTool.description}" escape="false"/>
-                            <p:commandLink styleClass="btn-explore" action="#{DatasetPage.explore(DatasetPage.datasetConfigureTool)}" oncomplete="PF('configureToolDialog').hide();"/>
                         </p>
+                        <div class="button-block">
+                            <p:commandLink styleClass="btn btn-default" action="#{DatasetPage.explore(DatasetPage.datasetConfigureTool)}" oncomplete="PF('configureToolDialog').hide();">
+                                <h:outputFormat value="#{bundle['continue']}"/>
+                            </p:commandLink>
+                            <button class="btn btn-link" onclick="PF('configureToolDialog').hide();" type="button">
+                                #{bundle.cancel}
+                            </button>
+                        </div>
                     </p:dialog>
                     <p:dialog id="citationsDialog" styleClass="smallPopUp" header="#{bundle['metrics.citations.dialog.header']}" widgetVar="citationsDialog" modal="true">
                         <p class="help-block">

From 8a3d4c23c726e70210c4248b262fe030ac30ede3 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 26 Sep 2023 16:07:11 -0400
Subject: [PATCH 456/922] better tool name in test, add URL to popup #9589

---
 .../harvard/iq/dataverse/api/ExternalToolsIT.java    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
index a9f6055fc9e..67fffadb488 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
@@ -277,13 +277,13 @@ public void testDatasetLevelToolConfigure() {
 
         String toolManifest = """
 {
-   "displayName": "Turbo Dataset Config",
-   "description": "Read/write access.",
+   "displayName": "Dataset Configurator",
+   "description": "Slices! Dices! <a href='https://docs.datasetconfigurator.com'>More info</a>.",
    "types": [
      "configure"
    ],
    "scope": "dataset",
-   "toolUrl": "http://datasettool1.com",
+   "toolUrl": "https://datasetconfigurator.com",
    "toolParameters": {
      "queryParameters": [
        {
@@ -301,15 +301,15 @@ public void testDatasetLevelToolConfigure() {
         addExternalTool.prettyPrint();
         addExternalTool.then().assertThat()
                 .statusCode(OK.getStatusCode())
-                .body("data.displayName", CoreMatchers.equalTo("Turbo Dataset Config"));
+                .body("data.displayName", CoreMatchers.equalTo("Dataset Configurator"));
 
         Response getExternalToolsByDatasetId = UtilIT.getExternalToolsForDataset(datasetId.toString(), "configure", apiToken);
         getExternalToolsByDatasetId.prettyPrint();
         getExternalToolsByDatasetId.then().assertThat()
-                .body("data[0].displayName", CoreMatchers.equalTo("Turbo Dataset Config"))
+                .body("data[0].displayName", CoreMatchers.equalTo("Dataset Configurator"))
                 .body("data[0].scope", CoreMatchers.equalTo("dataset"))
                 .body("data[0].types[0]", CoreMatchers.equalTo("configure"))
-                .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("http://datasettool1.com?datasetPid=" + datasetPid))
+                .body("data[0].toolUrlWithQueryParams", CoreMatchers.equalTo("https://datasetconfigurator.com?datasetPid=" + datasetPid))
                 .statusCode(OK.getStatusCode());
 
     }

From b76e7ee86e0f5e272c13dbc55166041ac6fa9e86 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 26 Sep 2023 16:10:05 -0400
Subject: [PATCH 457/922] fix typo about popup in docs #9589

---
 doc/sphinx-guides/source/api/external-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst
index ed68bb09ee0..d12e4b17549 100644
--- a/doc/sphinx-guides/source/api/external-tools.rst
+++ b/doc/sphinx-guides/source/api/external-tools.rst
@@ -88,7 +88,7 @@ Terminology
 
     displayName                  The **name** of the tool in the Dataverse installation web interface. For example, "Data Explorer".
 
-    description                  The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected the tool in a new tab in their browser. HTML is supported.
+    description                  The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected to the tool in a new tab in their browser. HTML is supported.
 
     scope                        Whether the external tool appears and operates at the **file** level or the **dataset** level. Note that a file level tool much also specify the type of file it operates on (see "contentType" below).
 

From 5840a068cc0947f2d90cf52e4d8c3e118fd23c0e Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 26 Sep 2023 16:12:50 -0400
Subject: [PATCH 458/922] example tool: open link in new tab #9589

---
 src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
index 67fffadb488..022747a3cdc 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ExternalToolsIT.java
@@ -278,7 +278,7 @@ public void testDatasetLevelToolConfigure() {
         String toolManifest = """
 {
    "displayName": "Dataset Configurator",
-   "description": "Slices! Dices! <a href='https://docs.datasetconfigurator.com'>More info</a>.",
+   "description": "Slices! Dices! <a href='https://docs.datasetconfigurator.com' target='_blank'>More info</a>.",
    "types": [
      "configure"
    ],

From a50776237a606e3b22bd67e2380abfc1e4303851 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 14 Aug 2023 21:57:00 +0200
Subject: [PATCH 459/922] build(deps): update to Maven Docker Plugin 0.43.3
 #9771

With this new version, a problem on M1 MACs not being able to build
with Docker because of a non-existing config file was worked around
by the DMP devs.

See also: https://github.com/fabric8io/docker-maven-plugin/issues/1678
---
 modules/dataverse-parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index c45d59e4f5f..cf136820430 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -198,7 +198,7 @@
         <pomchecker-maven-plugin.version>1.7.0</pomchecker-maven-plugin.version>
         
         <!-- Container related -->
-        <fabric8-dmp.version>0.43.0</fabric8-dmp.version>
+        <fabric8-dmp.version>0.43.3</fabric8-dmp.version>
     </properties>
     
     <pluginRepositories>

From c93b180df40c37e2f0ca17816d713cf8205f98ba Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Fri, 18 Aug 2023 09:19:34 +0200
Subject: [PATCH 460/922] build(deps): update to Maven Docker Plugin 0.43.4
 #9771

---
 modules/dataverse-parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index cf136820430..a7233cba164 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -198,7 +198,7 @@
         <pomchecker-maven-plugin.version>1.7.0</pomchecker-maven-plugin.version>
         
         <!-- Container related -->
-        <fabric8-dmp.version>0.43.3</fabric8-dmp.version>
+        <fabric8-dmp.version>0.43.4</fabric8-dmp.version>
     </properties>
     
     <pluginRepositories>

From f653c219d0ce6d7b1b1b3774b4820a05391c82d0 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 27 Sep 2023 10:33:21 +0100
Subject: [PATCH 461/922] Changed: dataset version download size calculation
 when ignoring original tab file sizes

---
 .../edu/harvard/iq/dataverse/DatasetServiceBean.java     | 9 ++++-----
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 ++++--
 .../command/impl/GetDatasetStorageSizeCommand.java       | 7 ++-----
 .../command/impl/GetDataverseStorageSizeCommand.java     | 2 +-
 .../java/edu/harvard/iq/dataverse/api/DatasetsIT.java    | 3 ++-
 5 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
index 4799502a6e3..30274efb384 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java
@@ -1015,12 +1015,12 @@ public void obtainPersistentIdentifiersForDatafiles(Dataset dataset) {
     }
 
     public long findStorageSize(Dataset dataset) throws IOException {
-        return findStorageSize(dataset, false, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
+        return findStorageSize(dataset, false, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
     }
 
 
     public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws IOException {
-        return findStorageSize(dataset, countCachedExtras, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
+        return findStorageSize(dataset, countCachedExtras, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
     }
 
     /**
@@ -1028,7 +1028,6 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
      *
      * @param dataset
      * @param countCachedExtras boolean indicating if the cached disposable extras should also be counted
-     * @param countOriginalTabularSize boolean indicating if the size of the stored original tabular files should also be counted, in addition to the main tab-delimited file size
      * @param mode String indicating whether we are getting the result for storage (entire dataset) or download version based
      * @param version optional param for dataset version
      * @return total size
@@ -1037,7 +1036,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras) throws I
      * default mode, the method doesn't need to access the storage system, as the
      * sizes of the main files are recorded in the database)
      */
-    public long findStorageSize(Dataset dataset, boolean countCachedExtras, boolean countOriginalTabularSize, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
+    public long findStorageSize(Dataset dataset, boolean countCachedExtras, GetDatasetStorageSizeCommand.Mode mode, DatasetVersion version) throws IOException {
         long total = 0L;
 
         if (dataset.isHarvested()) {
@@ -1063,7 +1062,7 @@ public long findStorageSize(Dataset dataset, boolean countCachedExtras, boolean
                 total += datafile.getFilesize();
 
                 if (!countCachedExtras) {
-                    if (datafile.isTabularData() && countOriginalTabularSize) {
+                    if (datafile.isTabularData()) {
                         // count the size of the stored original, in addition to the main tab-delimited file:
                         Long originalFileSize = datafile.getDataTable().getOriginalFileSize();
                         if (originalFileSize != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a39347ef64e..981cbced11e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2953,7 +2953,7 @@ public Response getMakeDataCountMetricCurrentMonth(@PathParam("id") String idSup
     @Path("{identifier}/storagesize")
     public Response getStorageSize(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @QueryParam("includeCached") boolean includeCached) {
         return response(req -> ok(MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.storage"),
-                execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), includeCached, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null)))), getRequestUser(crc));
+                execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), includeCached, GetDatasetStorageSizeCommand.Mode.STORAGE, null)))), getRequestUser(crc));
     }
 
     @GET
@@ -2966,7 +2966,9 @@ public Response getDownloadSize(@Context ContainerRequestContext crc,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
         return response(req -> {
-            Long datasetStorageSize = execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), false, !ignoreOriginalTabularSize, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers)));
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers);
+            Long datasetStorageSize = ignoreOriginalTabularSize ? DatasetUtil.getDownloadSizeNumeric(datasetVersion, false)
+                    : execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), false, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, datasetVersion));
             String message = MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.download"), datasetStorageSize);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
             jsonObjectBuilder.add("message", message);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
index eebb8dd9e00..09b33c4efc4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetStorageSizeCommand.java
@@ -32,7 +32,6 @@ public class GetDatasetStorageSizeCommand extends AbstractCommand<Long> {
 
     private final Dataset dataset;
     private final Boolean countCachedFiles;
-    private final Boolean countOriginalTabularSize;
     private final Mode mode;
     private final DatasetVersion version;
     
@@ -45,16 +44,14 @@ public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target) {
         super(aRequest, target);
         dataset = target;
         countCachedFiles = false;
-        countOriginalTabularSize = true;
         mode = Mode.DOWNLOAD;
         version = null;
     }
 
-    public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles, boolean countOriginalTabularSize, Mode mode, DatasetVersion version) {
+    public GetDatasetStorageSizeCommand(DataverseRequest aRequest, Dataset target, boolean countCachedFiles, Mode mode, DatasetVersion version) {
         super(aRequest, target);
         dataset = target;
         this.countCachedFiles = countCachedFiles;
-        this.countOriginalTabularSize = countOriginalTabularSize;
         this.mode = mode;
         this.version = version;
     }
@@ -69,7 +66,7 @@ public Long execute(CommandContext ctxt) throws CommandException {
         logger.fine("getDataverseStorageSize called on " + dataset.getDisplayName());
 
         try {
-            return ctxt.datasets().findStorageSize(dataset, countCachedFiles, countOriginalTabularSize, mode, version);
+            return ctxt.datasets().findStorageSize(dataset, countCachedFiles, mode, version);
         } catch (IOException ex) {
             throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.datasize.ioerror"), this);
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java
index 9f93f6747ea..57912a6b4bd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDataverseStorageSizeCommand.java
@@ -59,7 +59,7 @@ public Long execute(CommandContext ctxt) throws CommandException {
             }
             
             try {
-                total += ctxt.datasets().findStorageSize(dataset, countCachedFiles, true, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
+                total += ctxt.datasets().findStorageSize(dataset, countCachedFiles, GetDatasetStorageSizeCommand.Mode.STORAGE, null);
             } catch (IOException ex) {
                 throw new CommandException(BundleUtil.getStringFromBundle("dataverse.datasize.ioerror"), this);
             }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 929882fe95a..580a1edb6f2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3679,7 +3679,8 @@ public void getDownloadSize() throws IOException {
         assertTrue(actualSizeIgnoringOriginalTabularSizes > expectedTextFilesStorageSize);
 
         // Get the size including the original tabular file sizes
-        int expectedSizeIncludingOriginalTabularSizes = tabularOriginalSize + actualSizeIgnoringOriginalTabularSizes;
+        int tabularProcessedSize = actualSizeIgnoringOriginalTabularSizes - expectedTextFilesStorageSize;
+        int expectedSizeIncludingOriginalTabularSizes = tabularOriginalSize + tabularProcessedSize + expectedTextFilesStorageSize;
 
         getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())

From 9d10b99cdbb3487e08a308e0e6f1de7ff69cf913 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 27 Sep 2023 10:40:23 +0100
Subject: [PATCH 462/922] Added: #9958 release notes

---
 .../9958-dataset-api-downloadsize-ignore-tabular-size.md       | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md

diff --git a/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md b/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
new file mode 100644
index 00000000000..73b27a1a581
--- /dev/null
+++ b/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
@@ -0,0 +1,3 @@
+Added a new optional query parameter "ignoreOriginalTabularSize" to the "getDownloadSize" API endpoint ("api/datasets/{identifier}/versions/{versionId}/downloadsize").
+
+If set to true, the endpoint will return the download size ignoring the original tabular file sizes.

From 9710c79432cbc30a1f3222a2df2e423f6040ed0a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 27 Sep 2023 10:46:05 +0100
Subject: [PATCH 463/922] Added: mentioned ignoreOriginalTabularSize query
 parameter in the docs for /downloadsize API endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 663051c0884..169b950dc74 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1797,6 +1797,8 @@ The fully expanded example above (without environment variables) looks like this
 The size of all files available for download will be returned. 
 If :draft is passed as versionId the token supplied must have permission to view unpublished drafts. A token is not required for published datasets. Also restricted files will be included in this total regardless of whether the user has access to download the restricted file(s).
 
+There is an optional query parameter ``ignoreOriginalTabularSize`` which, if set to true, the endpoint will return the download size ignoring the sizes of the original tabular files. Otherwise, both the original and the processed size will be included in the count for tabular files.
+
 Submit a Dataset for Review
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 4aa34ffb417039b8132070270a246f8e4b4fedd3 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 27 Sep 2023 10:50:42 +0100
Subject: [PATCH 464/922] Added: ignoreOriginalTabularSize query param usage
 example to the docs

---
 doc/sphinx-guides/source/api/native-api.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 169b950dc74..0f77aeba580 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1799,6 +1799,12 @@ If :draft is passed as versionId the token supplied must have permission to view
 
 There is an optional query parameter ``ignoreOriginalTabularSize`` which, if set to true, the endpoint will return the download size ignoring the sizes of the original tabular files. Otherwise, both the original and the processed size will be included in the count for tabular files.
 
+Usage example:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?ignoreOriginalTabularSize=true"
+
 Submit a Dataset for Review
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 6c7826f434399648194cd7ac54e232411a24d161 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 27 Sep 2023 10:27:27 -0400
Subject: [PATCH 465/922] Consolidate flyway scripts

---
 .../migration/V5.13.0.3__guestbook-on-request.sql   | 12 ------------
 .../V6.0.0.1__9599-guestbook-at-request.sql         | 13 +++++++++++++
 2 files changed, 13 insertions(+), 12 deletions(-)
 delete mode 100644 src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql

diff --git a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql b/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
deleted file mode 100644
index 7e05e792858..00000000000
--- a/src/main/resources/db/migration/V5.13.0.3__guestbook-on-request.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
-ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
-ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
-ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT;
-ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_guestbookresponse;
-ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);
-DROP INDEX IF EXISTS created_requests;
-CREATE UNIQUE INDEX created_requests ON fileaccessrequests (datafile_id, authenticated_user_id) WHERE request_state='CREATED';
-
-ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS guestbookatrequest bool;
-ALTER TABLE dataset ADD COLUMN IF NOT EXISTS guestbookatrequest bool;
diff --git a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
index 109963d8382..c90ee4a5329 100644
--- a/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
+++ b/src/main/resources/db/migration/V6.0.0.1__9599-guestbook-at-request.sql
@@ -1,3 +1,16 @@
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS request_state VARCHAR(64);
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS id SERIAL;
+ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fileaccessrequests_pkey;
+ALTER TABLE fileaccessrequests ADD CONSTRAINT fileaccessrequests_pkey PRIMARY KEY (id);
+ALTER TABLE fileaccessrequests ADD COLUMN IF NOT EXISTS guestbookresponse_id INT;
+ALTER TABLE fileaccessrequests DROP CONSTRAINT IF EXISTS fk_fileaccessrequests_guestbookresponse;
+ALTER TABLE fileaccessrequests ADD CONSTRAINT fk_fileaccessrequests_guestbookresponse FOREIGN KEY (guestbookresponse_id) REFERENCES guestbookresponse(id);
+DROP INDEX IF EXISTS created_requests;
+CREATE UNIQUE INDEX created_requests ON fileaccessrequests (datafile_id, authenticated_user_id) WHERE request_state='CREATED';
+
+ALTER TABLE dataverse ADD COLUMN IF NOT EXISTS guestbookatrequest bool;
+ALTER TABLE dataset ADD COLUMN IF NOT EXISTS guestbookatrequest bool;
+
 ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS eventtype VARCHAR(255);
 ALTER TABLE guestbookresponse ADD COLUMN IF NOT EXISTS sessionid VARCHAR(255);
 

From c9c6cf26a1764bb5c409c4d25571984d0e5fbf80 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 27 Sep 2023 10:32:44 -0400
Subject: [PATCH 466/922] Add null check to avoid future issues

---
 .../java/edu/harvard/iq/dataverse/DatasetPage.java  | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 7cb5bfa3850..7dba8af3fdc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -2863,6 +2863,12 @@ public void sort() {
     public String refresh() {
         logger.fine("refreshing");
 
+        //In v5.14, versionId was null here. In 6.0, it appears not to be.
+        //This check is to handle the null if it reappears/occurs under other circumstances
+        if(versionId==null) {
+            logger.fine("versionId was null in refresh");
+            versionId = workingVersion.getId();
+        }
         //dataset = datasetService.find(dataset.getId());
         dataset = null;
         workingVersion = null; 
@@ -2872,10 +2878,9 @@ public String refresh() {
         DatasetVersionServiceBean.RetrieveDatasetVersionResponse retrieveDatasetVersionResponse = null;
 
         if (versionId != null) {
-            // versionId must have been set by now, in the init() method, 
-            // regardless of how the page was originally called - by the dataset
-            // database id, by the persistent identifier, or by the db id of
-            // the version. 
+            // versionId must have been set by now (see null check above), in the init()
+            // method, regardless of how the page was originally called - by the dataset
+            // database id, by the persistent identifier, or by the db id of the version.
             this.workingVersion = datasetVersionService.findDeep(versionId);
             dataset = workingVersion.getDataset();
         } 

From ba2dd8400c128cefd22f7d83d52771760d477905 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 27 Sep 2023 10:39:26 -0400
Subject: [PATCH 467/922] warn if null cases still occur

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 7dba8af3fdc..74064f20893 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -2866,7 +2866,7 @@ public String refresh() {
         //In v5.14, versionId was null here. In 6.0, it appears not to be.
         //This check is to handle the null if it reappears/occurs under other circumstances
         if(versionId==null) {
-            logger.fine("versionId was null in refresh");
+            logger.warning("versionId was null in refresh");
             versionId = workingVersion.getId();
         }
         //dataset = datasetService.find(dataset.getId());
@@ -6150,6 +6150,9 @@ public String getWebloaderUrlForDataset(Dataset d) {
 
     public String getSignpostingLinkHeader() {
         if ((workingVersion==null) || (!workingVersion.isReleased())) {
+            if(workingVersion==null) {
+                logger.warning("workingVersion was null in getSignpostingLinkHeader");
+            }
             return null;
         }
         if (signpostingLinkHeader == null) {

From 16322dfd2ae3f288d0f5505f9e892b3370cb751a Mon Sep 17 00:00:00 2001
From: Kevin Condon <kcondon@hmdc.harvard.edu>
Date: Wed, 27 Sep 2023 14:35:30 -0400
Subject: [PATCH 468/922] Create 9002_allow_direct_upload_setting.md

Adding release notes snippet, directly taken from installation guide.
---
 doc/release-notes/9002_allow_direct_upload_setting.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/9002_allow_direct_upload_setting.md

diff --git a/doc/release-notes/9002_allow_direct_upload_setting.md b/doc/release-notes/9002_allow_direct_upload_setting.md
new file mode 100644
index 00000000000..581da5592da
--- /dev/null
+++ b/doc/release-notes/9002_allow_direct_upload_setting.md
@@ -0,0 +1,3 @@
+A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
+By default, Dataverse supports uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`Adding the Uploaded file to the Dataset <direct-add-to-dataset-api>` API call (described in the :doc:`/developers/s3-direct-upload-api` page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.

From 3ecd118d97fb3e6869e4caab9f21b060a3130d2a Mon Sep 17 00:00:00 2001
From: Kevin Condon <kcondon@hmdc.harvard.edu>
Date: Wed, 27 Sep 2023 15:46:44 -0400
Subject: [PATCH 469/922] Update
 doc/release-notes/9002_allow_direct_upload_setting.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/release-notes/9002_allow_direct_upload_setting.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/doc/release-notes/9002_allow_direct_upload_setting.md b/doc/release-notes/9002_allow_direct_upload_setting.md
index 581da5592da..1e76ed4ad47 100644
--- a/doc/release-notes/9002_allow_direct_upload_setting.md
+++ b/doc/release-notes/9002_allow_direct_upload_setting.md
@@ -1,3 +1,5 @@
-A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
-By default, Dataverse supports uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
-With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`Adding the Uploaded file to the Dataset <direct-add-to-dataset-api>` API call (described in the :doc:`/developers/s3-direct-upload-api` page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+A Dataverse installation can be now be configured to allow out-of-band upload by setting the `dataverse.files.<id>.upload-out-of-band` JVM option to `true`.
+
+By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.

From 448ae448ff1cb36f10b30449694126a866c28643 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Thu, 28 Sep 2023 11:13:52 +0100
Subject: [PATCH 470/922] Added: JSON payload to curl examples for Deaccession
 Dataset docs

---
 doc/sphinx-guides/source/api/native-api.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 663051c0884..01a681cfb6a 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1375,14 +1375,15 @@ Given a version of a dataset, updates its status to deaccessioned.
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
   export VERSIONID=1.0
+  export JSON='{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession" -d "$JSON"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -d '{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
 
 Set Citation Date Field Type for a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 023c8b22c1f01da318cb87072f1c0637ac059980 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Thu, 28 Sep 2023 14:33:45 +0200
Subject: [PATCH 471/922] doc clarification

---
 doc/sphinx-guides/source/api/native-api.rst | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 56d245f97c0..1cc462ee02a 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -525,10 +525,16 @@ Submit Incomplete Dataset
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 **Note:** This feature requires :ref:`dataverse.api.allow-incomplete-metadata` to be enabled and your Solr
-Schema to be up-to-date with the ``datasetValid`` field.
+Schema to be up-to-date with the ``datasetValid`` field. If not done yet with the version upgrade, you will
+also need to reindex all dataset after enabling the :ref:`dataverse.api.allow-incomplete-metadata` feature.
 
 Providing a ``.../datasets?doNotValidate=true`` query parameter turns off the validation of metadata.
-In this case, only the "Author Name" is required. For example, a minimal JSON file would look like this:
+In this situation, only the "Author Name" is required, except for the case when the setting :ref:`:MetadataLanguages`
+is configured and the value of "Dataset Metadata Language" setting of a collection is left with the default
+"Chosen at Dataset Creation" value. In that case, a language that is a part of the :ref:`:MetadataLanguages` list must be
+declared in the incomplete dataset.
+
+For example, a minimal JSON file, without the language specification, would look like this:
 
 .. code-block:: json
   :name: dataset-incomplete.json

From 3d7a2dee22636a9aaba0f5c3a144b3a2b05e25c3 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 28 Sep 2023 10:52:24 -0400
Subject: [PATCH 472/922] #9913 show TOA on manage templates preview

---
 src/main/webapp/dataset-license-terms.xhtml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml
index c5958697a20..b998442f2aa 100644
--- a/src/main/webapp/dataset-license-terms.xhtml
+++ b/src/main/webapp/dataset-license-terms.xhtml
@@ -238,7 +238,7 @@
                 </p:fragment>
             </div>
         </div>
-       <div class="panel panel-default" jsf:rendered="#{!empty editMode or DatasetPage.restrictedFileCount > 0 
+       <div class="panel panel-default" jsf:rendered="#{ managePage==true or !empty editMode or DatasetPage.restrictedFileCount > 0 
                                                         or !empty termsOfUseAndAccess.termsOfAccess
                                                         or !empty termsOfUseAndAccess.dataAccessPlace 
                                         or !empty termsOfUseAndAccess.originalArchive or !empty termsOfUseAndAccess.availabilityStatus 
@@ -267,7 +267,7 @@
                                </div>
                            </ui:fragment>
                        </div>
-                       <div class="form-group" jsf:rendered="#{!empty termsOfUseAndAccess.termsOfAccess and DatasetPage.restrictedFileCount > 0}">
+                       <div class="form-group" jsf:rendered="#{(!empty termsOfUseAndAccess.termsOfAccess and DatasetPage.restrictedFileCount > 0) or managePage==true}">
                            <label for="datasetForm:tabView:metadata_TermsAccessText" class="col-sm-3 control-label">
                                #{bundle['file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess']}
                                <span class="glyphicon glyphicon-question-sign tooltip-icon"

From d08f4f183c9652f5e83aa31089066e9760dc053b Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 28 Sep 2023 10:57:43 -0400
Subject: [PATCH 473/922] #9913 add style class

---
 src/main/webapp/manage-templates.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/manage-templates.xhtml b/src/main/webapp/manage-templates.xhtml
index c9841ace8e8..879cf9e55c2 100644
--- a/src/main/webapp/manage-templates.xhtml
+++ b/src/main/webapp/manage-templates.xhtml
@@ -139,7 +139,7 @@
                         </p:dataTable>
                     </div>
                     <p:dialog id="deleteTemplateConfirm" header="#{bundle['dataset.manageTemplates.tab.action.btn.delete.dialog.header']}" 
-                              widgetVar="deleteConfirmation" modal="true" focus="contDeleteTemplateBtn">
+                              styleClass="largePopUp" widgetVar="deleteConfirmation" modal="true" focus="contDeleteTemplateBtn">
                         <p class="help-block">
                             <span class="text-warning"><span class="glyphicon glyphicon-warning-sign"/> &#160;<h:outputText value="#{bundle['dataset.manageTemplates.tab.action.btn.delete.dialog.tip']}" /></span>
                         </p>

From 28df26c7d9f88b4c594e21bfcf1fbd4b695dc1b1 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 28 Sep 2023 16:34:20 -0400
Subject: [PATCH 474/922] #9913 make request access true on create template

also fix various render issues.
---
 .../harvard/iq/dataverse/TemplatePage.java    |  1 +
 src/main/webapp/dataset-license-terms.xhtml   | 25 +++++++++++--------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java b/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java
index fff520fd259..44070dcbb41 100644
--- a/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/TemplatePage.java
@@ -148,6 +148,7 @@ public String init() {
             editMode = TemplatePage.EditMode.CREATE;
             template = new Template(this.dataverse, settingsWrapper.getSystemMetadataBlocks());
             TermsOfUseAndAccess terms = new TermsOfUseAndAccess();
+            terms.setFileAccessRequest(true);
             terms.setTemplate(template);
             terms.setLicense(licenseServiceBean.getDefault());
             template.setTermsOfUseAndAccess(terms);
diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml
index b998442f2aa..c54d94442ea 100644
--- a/src/main/webapp/dataset-license-terms.xhtml
+++ b/src/main/webapp/dataset-license-terms.xhtml
@@ -7,6 +7,13 @@
                 xmlns:o="http://omnifaces.org/ui"
                 xmlns:jsf="http://xmlns.jcp.org/jsf">
     <!-- TERMS -->
+    <ui:param name="toaNotEmpty" value="#{!empty termsOfUseAndAccess.termsOfAccess
+                                        or !empty termsOfUseAndAccess.dataAccessPlace 
+                                        or !empty termsOfUseAndAccess.originalArchive or !empty termsOfUseAndAccess.availabilityStatus 
+                                        or !empty termsOfUseAndAccess.contactForAccess or !empty termsOfUseAndAccess.sizeOfCollection 
+                                        or !empty termsOfUseAndAccess.studyCompletion
+                                        or termsOfUseAndAccess.fileAccessRequest}"/>
+    
     <div class="text-right margin-bottom" 
          jsf:rendered="#{dataverseSession.user.authenticated and empty editMode and !widgetWrapper.widgetView
                          and permissionsWrapper.canIssueUpdateDatasetCommand(DatasetPage.dataset)}">
@@ -238,17 +245,13 @@
                 </p:fragment>
             </div>
         </div>
-       <div class="panel panel-default" jsf:rendered="#{ managePage==true or !empty editMode or DatasetPage.restrictedFileCount > 0 
-                                                        or !empty termsOfUseAndAccess.termsOfAccess
-                                                        or !empty termsOfUseAndAccess.dataAccessPlace 
-                                        or !empty termsOfUseAndAccess.originalArchive or !empty termsOfUseAndAccess.availabilityStatus 
-                                        or !empty termsOfUseAndAccess.contactForAccess or !empty termsOfUseAndAccess.sizeOfCollection 
-                                        or !empty termsOfUseAndAccess.studyCompletion}">
+       <div class="panel panel-default" jsf:rendered="#{managePage==true or !empty editMode or (datasetPage == true and DatasetPage.restrictedFileCount > 0) 
+                                                        or toaNotEmpty}">
            <div data-toggle="collapse" data-target="#panelCollapseTOA" class="panel-heading text-info">
                <h:outputText value="#{publicStore ? bundle['file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess'] : bundle['file.dataFilesTab.terms.list.termsOfAccess.header']}"/>
                &#160;<span class="glyphicon glyphicon-chevron-up"/>
            </div>
-           <div id="panelCollapseTOA" class="#{!datasetPage or DatasetPage.hasValidTermsOfAccess ? 'collapse in' : 'collapse'}">
+           <div id="panelCollapseTOA" class="panel-collapse #{!empty editMode or (datasetPage == false and toaNotEmpty )  or (datasetPage == true and DatasetPage.hasValidTermsOfAccess) ? 'collapse in' : 'collapse'}">
                <div class="panel-body">
                    <ui:fragment rendered="#{empty editMode}">
                        <div class="form-group" jsf:rendered="#{!publicStore  and datasetPage== true and DatasetPage.hasRestrictedFiles}">
@@ -267,7 +270,7 @@
                                </div>
                            </ui:fragment>
                        </div>
-                       <div class="form-group" jsf:rendered="#{(!empty termsOfUseAndAccess.termsOfAccess and DatasetPage.restrictedFileCount > 0) or managePage==true}">
+                       <div class="form-group" jsf:rendered="#{!empty termsOfUseAndAccess.termsOfAccess and (DatasetPage.restrictedFileCount > 0 or managePage==true)}">
                            <label for="datasetForm:tabView:metadata_TermsAccessText" class="col-sm-3 control-label">
                                #{bundle['file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess']}
                                <span class="glyphicon glyphicon-question-sign tooltip-icon"
@@ -277,18 +280,18 @@
                                <p><h:outputText id="metadata_TermsAccessText" value="#{MarkupChecker:sanitizeBasicHTML(termsOfUseAndAccess.termsOfAccess)}" escape="false"/></p>
                            </div>
                        </div>
-                       <div class="form-group" jsf:rendered="#{!publicStore and ( datasetPage == true and DatasetPage.hasRestrictedFiles)}">
+                       <div class="form-group" jsf:rendered="#{!publicStore and ((datasetPage == true and DatasetPage.hasRestrictedFiles) or managePage==true)}">
                            <label for="datasetForm:tabView:metadata_RequestAccessSelected" class="col-sm-3 control-label">
                                #{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess']}
                                <span class="glyphicon glyphicon-question-sign tooltip-icon"
                                      data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess.title']}"></span>
                            </label>
                            <div id="metadata_RequestAccessSelected" class="col-sm-9">
-                               <p jsf:rendered="#{!DatasetPage.workingVersion.draft}">
+                               <p jsf:rendered="#{datasetPage == true and !DatasetPage.workingVersion.draft}">
                                    <h:outputText value="#{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess.request']}" rendered="#{DatasetPage.dataset.fileAccessRequest}"/>
                                    <h:outputText value="#{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess.notRequest']}" rendered="#{!DatasetPage.dataset.fileAccessRequest}"/>
                                </p>
-                               <p jsf:rendered="#{DatasetPage.workingVersion.draft}">
+                               <p jsf:rendered="#{(datasetPage == true and DatasetPage.workingVersion.draft) or managePage==true}">
                                    <h:outputText value="#{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess.request']}" rendered="#{termsOfUseAndAccess.fileAccessRequest}"/>
                                    <h:outputText value="#{bundle['file.dataFilesTab.terms.list.termsOfAccess.requestAccess.notRequest']}" rendered="#{!termsOfUseAndAccess.fileAccessRequest}"/>
                                </p>

From 29e56bbe5a3e0ec8f80d98e8d858665d6b63250d Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 28 Sep 2023 17:17:16 -0400
Subject: [PATCH 475/922] saving temp. changes

---
 .../edu/harvard/iq/dataverse/EditDatafilesPage.java    | 10 ++--------
 .../engine/command/impl/CreateNewDataFilesCommand.java |  6 +++++-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index 4144112f997..7b8f0355f4c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -2187,16 +2187,10 @@ public void handleExternalUpload() {
                 List<DataFile> datafiles = new ArrayList<>();
 
                 // -----------------------------------------------------------
-                // Send it through the ingest service
+                // Execute the CreateNewDataFiles command:
                 // -----------------------------------------------------------
                 try {
-
-                    // Note: A single uploaded file may produce multiple datafiles - 
-                    // for example, multiple files can be extracted from an uncompressed
-                    // zip file.
-                    //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");
-                    ///CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, checksumType, systemConfig);
-                    
+  
                     Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, userStorageQuota, checksumValue, checksumType);
                     CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                     datafiles = createDataFilesResult.getDataFiles();
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index e7fc30bdd80..6fa3c721c38 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -648,15 +648,19 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
         // if we were unable to unpack an uploaded file, etc.), we'll just 
         // create and return a single DataFile:
         File newFile = null;
+        long fileSize = -1; 
+        
         if (tempFile != null) {
             newFile = tempFile.toFile();
+            fileSize = newFile.length();
+        } else {
+            // @todo! What do we do if this is direct upload?? where does the size come from? 
         }
         
         // We have already checked that this file does not exceed the individual size limit; 
         // but if we are processing it as is, as a single file, we need to check if 
         // its size does not go beyond the allocated storage quota (if specified):
         
-        long fileSize = newFile.length();
         
         if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
             try {

From 7ab736c05a2aac0b7137861d46a6224ed561e228 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 29 Sep 2023 15:29:46 -0400
Subject: [PATCH 476/922] Fixes direct upload from the UI. (#9361)

---
 .../iq/dataverse/EditDatafilesPage.java       | 21 +++++-
 .../impl/CreateNewDataFilesCommand.java       | 66 +++++++++++--------
 2 files changed, 57 insertions(+), 30 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index 7b8f0355f4c..a942830b19e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -2068,7 +2068,7 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
                 // dataset that does not yet exist in the database. We must 
                 // use the version of the Create New Files constructor that takes
                 // the parent Dataverse as the extra argument:
-                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null, null, workingVersion.getDataset().getOwner());
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null, null, null, workingVersion.getDataset().getOwner());
             } else {
                 cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null);
             }
@@ -2168,6 +2168,11 @@ public void handleExternalUpload() {
                 - Max size NOT specified in db: default is unlimited
                 - Max size specified in db: check too make sure file is within limits
             // ---------------------------- */
+            /**
+             * @todo: this size check is probably redundant here, since the new
+             * CreateNewFilesCommand is going to perform it (and the quota 
+             * checks too, if enabled
+             */
             if ((!this.isUnlimitedUploadFileSize()) && (fileSize > this.getMaxFileUploadSizeInBytes())) {
                 String warningMessage = "Uploaded file \"" + fileName + "\" exceeded the limit of " + fileSize + " bytes and was not uploaded.";
                 sio.delete();
@@ -2189,9 +2194,21 @@ public void handleExternalUpload() {
                 // -----------------------------------------------------------
                 // Execute the CreateNewDataFiles command:
                 // -----------------------------------------------------------
+                
+                Dataverse parent = null; 
+                
+                if (mode == FileEditMode.CREATE) {
+                    // This is a file upload in the context of creating a brand new
+                    // dataset that does not yet exist in the database. We must 
+                    // pass the parent Dataverse to the CreateNewFiles command
+                    // constructor. The RequiredPermission on the command in this 
+                    // scenario = Permission.AddDataset on the parent dataverse.
+                    parent = workingVersion.getDataset().getOwner();
+                }
+                
                 try {
   
-                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, userStorageQuota, checksumValue, checksumType);
+                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, userStorageQuota, checksumValue, checksumType, fileSize, parent);
                     CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                     datafiles = createDataFilesResult.getDataFiles();
                     Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 6fa3c721c38..ac701da1be9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -74,34 +74,29 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private final InputStream inputStream;
     private final String fileName;
     private final String suppliedContentType; 
+    private final UserStorageQuota quota;
+    // parent Dataverse must be specified when the command is called on Create 
+    // of a new dataset that does not exist in the database yet (for the purposes
+    // of authorization - see getRequiredPermissions() below):
+    private final Dataverse parentDataverse;
+    // With Direct Upload the following values already exist and are passed to the command:
     private final String newStorageIdentifier; 
     private final String newCheckSum; 
     private DataFile.ChecksumType newCheckSumType;
-    private final Dataverse dataverse;
-    private final UserStorageQuota quota;
+    private final Long newFileSize;
 
     public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum) {
         this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, null);
     }
     
     public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType) {
-        super(aRequest, version.getDataset());
-        
-        this.version = version;
-        this.inputStream = inputStream;
-        this.fileName = fileName;
-        this.suppliedContentType = suppliedContentType; 
-        this.newStorageIdentifier = newStorageIdentifier; 
-        this.newCheckSum = newCheckSum; 
-        this.newCheckSumType = newCheckSumType;
-        this.dataverse = null; 
-        this.quota = quota;
+        this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, null, null);
     }
     
     // This version of the command must be used when files are created in the 
     // context of creating a brand new dataset (from the Add Dataset page):
     
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Dataverse dataverse) {
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize, Dataverse dataverse) {
         super(aRequest, dataverse);
         
         this.version = version;
@@ -111,8 +106,9 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
         this.newStorageIdentifier = newStorageIdentifier; 
         this.newCheckSum = newCheckSum; 
         this.newCheckSumType = newCheckSumType;
-        this.dataverse = dataverse;
-        this.quota = quota; 
+        this.parentDataverse = dataverse;
+        this.quota = quota;
+        this.newFileSize = newFileSize;
     }
     
 
@@ -654,7 +650,15 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             newFile = tempFile.toFile();
             fileSize = newFile.length();
         } else {
-            // @todo! What do we do if this is direct upload?? where does the size come from? 
+            // If this is a direct upload, and therefore no temp file associated 
+            // with it, the file size must be explicitly passed to the command 
+            // (note that direct upload relies on knowing the size of the file 
+            // that's being uploaded in advance).
+            if (newFileSize != null) {
+                fileSize = newFileSize;
+            } else {
+                throw new CommandExecutionException("File size must be explicitly specified when creating DataFiles with Direct Upload", this);
+            }
         }
         
         // We have already checked that this file does not exceed the individual size limit; 
@@ -663,26 +667,32 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
         
         
         if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
-            try {
-                tempFile.toFile().delete();
-            } catch (Exception ex) {
-                // ignore - but log a warning
-                logger.warning("Could not remove temp file " + tempFile.getFileName());
+            if (newFile != null) {
+                // Remove the temp. file, if this is a non-direct upload. 
+                // If this is a direct upload, it will be a responsibility of the 
+                // component calling the command to remove the file that may have
+                // already been saved in the S3 volume. 
+                try {
+                    newFile.delete();
+                } catch (Exception ex) {
+                    // ignore - but log a warning
+                    logger.warning("Could not remove temp file " + tempFile.getFileName());
+                }
             }
             throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(fileSize), bytesToHumanReadable(storageQuotaLimit)), this);
         } 
         
         DataFile datafile = FileUtil.createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum);
-        File f = null;
-        if (tempFile != null) {
-            f = tempFile.toFile();
-        }
-        if (datafile != null && ((f != null) || (newStorageIdentifier != null))) {
+
+        if (datafile != null && ((newFile != null) || (newStorageIdentifier != null))) {
 
             if (warningMessage != null) {
                 createIngestFailureReport(datafile, warningMessage);
                 datafile.SetIngestProblem();
             }
+            if (datafile.getFilesize() < 0) {
+                datafile.setFilesize(fileSize);
+            }
             datafiles.add(datafile);
 
             // Update quota (may not be necessary in the context of direct upload - ?)
@@ -701,7 +711,7 @@ public Map<String, Set<Permission>> getRequiredPermissions() {
 
         ret.put("", new HashSet<>());
         
-        if (dataverse != null) {
+        if (parentDataverse != null) {
             // The command is called in the context of uploading files on 
             // create of a new dataset
             ret.get("").add(Permission.AddDataset);

From 7881919ede378886fe6283086a02a4dfea8989a8 Mon Sep 17 00:00:00 2001
From: qqmyers <jim.myers@computer.org>
Date: Fri, 29 Sep 2023 16:07:09 -0400
Subject: [PATCH 477/922] Update native-api.rst

misplaced quote marks
---
 doc/sphinx-guides/source/api/native-api.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 176cb36e288..b9b0bd32aec 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2283,7 +2283,7 @@ To set the behavior for this dataset:
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
 
-  curl -X PUT "-H:X-Dataverse-key:$API_TOKEN" -H Content-type:application/json -d true "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+  curl -X PUT -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json -d true "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
 
 
   This example uses true to set the behavior to guestbook at request. Note that this call will return a 403/Forbidden response if guestbook at request functionality is not enabled for this Dataverse instance.
@@ -2296,7 +2296,7 @@ The API can also be used to reset the dataset to use the default/inherited value
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
 
-  curl -X DELETE "-H:X-Dataverse-key:$API_TOKEN" -H Content-type:application/json "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+  curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
 
 
 

From 907fd4024c8df2218764fd0902d1242a37726f7e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 10:48:36 +0100
Subject: [PATCH 478/922] Changed: using query-based implementation for files
 download size

---
 .../DatasetVersionFilesServiceBean.java       | 57 +++++++++++++++++++
 .../harvard/iq/dataverse/api/Datasets.java    | 12 +++-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 30 +++++++---
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  4 +-
 4 files changed, 89 insertions(+), 14 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index a547a216ad5..66e0ec5b5fe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.QDataFileCategory;
+import edu.harvard.iq.dataverse.QDataTable;
 import edu.harvard.iq.dataverse.QDvObject;
 import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
@@ -36,6 +37,7 @@ public class DatasetVersionFilesServiceBean implements Serializable {
     private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
     private final QDvObject dvObject = QDvObject.dvObject;
     private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
+    private final QDataTable dataTable = QDataTable.dataTable;
 
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
@@ -51,6 +53,19 @@ public enum DataFileAccessStatus {
         Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
     }
 
+    /**
+     * Mode to base the search in {@link DatasetVersionFilesServiceBean#getFilesDownloadSize(DatasetVersion, FileDownloadSizeMode)}
+     * <p>
+     * All: Includes both archival and original sizes for tabular files
+     * Archival: Includes only the archival size for tabular files
+     * Original: Includes only the original size for tabular files
+     * <p>
+     * All the modes include archival sizes for non-tabular files
+     */
+    public enum FileDownloadSizeMode {
+        All, Original, Archival
+    }
+
     /**
      * Given a DatasetVersion, returns its total file metadata count
      *
@@ -159,6 +174,23 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         return baseQuery.fetch();
     }
 
+    /**
+     * Returns the total download size of all files for a particular DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param mode           a FileDownloadSizeMode to base the search on
+     * @return long value of total file download size
+     */
+    public long getFilesDownloadSize(DatasetVersion datasetVersion, FileDownloadSizeMode mode) {
+        return switch (mode) {
+            case All ->
+                    Long.sum(getOriginalTabularFilesSize(datasetVersion), getArchivalFilesSize(datasetVersion, false));
+            case Original ->
+                    Long.sum(getOriginalTabularFilesSize(datasetVersion), getArchivalFilesSize(datasetVersion, true));
+            case Archival -> getArchivalFilesSize(datasetVersion, false);
+        };
+    }
+
     private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<DataFileAccessStatus, Long> totalCounts, DataFileAccessStatus dataFileAccessStatus) {
         long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus);
         if (fileMetadataCount > 0) {
@@ -230,4 +262,29 @@ private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> qu
                 break;
         }
     }
+
+    private long getOriginalTabularFilesSize(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        Long result = queryFactory
+                .from(fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                .from(dataTable)
+                .where(fileMetadata.dataFile.dataTables.isNotEmpty().and(dataTable.dataFile.eq(fileMetadata.dataFile)))
+                .select(dataTable.originalFileSize.sum()).fetchFirst();
+        return (result == null) ? 0 : result;
+    }
+
+    private long getArchivalFilesSize(DatasetVersion datasetVersion, boolean ignoreTabular) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        JPAQuery<?> baseQuery = queryFactory
+                .from(fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+        Long result;
+        if (ignoreTabular) {
+            result = baseQuery.where(fileMetadata.dataFile.dataTables.isEmpty()).select(fileMetadata.dataFile.filesize.sum()).fetchFirst();
+        } else {
+            result = baseQuery.select(fileMetadata.dataFile.filesize.sum()).fetchFirst();
+        }
+        return (result == null) ? 0 : result;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 981cbced11e..80a2dac9568 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2962,13 +2962,19 @@ public Response getStorageSize(@Context ContainerRequestContext crc, @PathParam(
     public Response getDownloadSize(@Context ContainerRequestContext crc,
                                     @PathParam("identifier") String dvIdtf,
                                     @PathParam("versionId") String version,
-                                    @QueryParam("ignoreOriginalTabularSize") boolean ignoreOriginalTabularSize,
+                                    @QueryParam("mode") String mode,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
+
         return response(req -> {
+            DatasetVersionFilesServiceBean.FileDownloadSizeMode fileDownloadSizeMode;
+            try {
+                fileDownloadSizeMode = mode != null ? DatasetVersionFilesServiceBean.FileDownloadSizeMode.valueOf(mode) : DatasetVersionFilesServiceBean.FileDownloadSizeMode.All;
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid mode: " + mode);
+            }
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers);
-            Long datasetStorageSize = ignoreOriginalTabularSize ? DatasetUtil.getDownloadSizeNumeric(datasetVersion, false)
-                    : execCommand(new GetDatasetStorageSizeCommand(req, findDatasetOrDie(dvIdtf), false, GetDatasetStorageSizeCommand.Mode.DOWNLOAD, datasetVersion));
+            long datasetStorageSize = datasetVersionFilesServiceBean.getFilesDownloadSize(datasetVersion, fileDownloadSizeMode);
             String message = MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.download"), datasetStorageSize);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
             jsonObjectBuilder.add("message", message);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 580a1edb6f2..189cf3a6f5a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3632,7 +3632,7 @@ public void deaccessionDataset() {
     }
 
     @Test
-    public void getDownloadSize() throws IOException {
+    public void getDownloadSize() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3658,7 +3658,8 @@ public void getDownloadSize() throws IOException {
 
         int expectedTextFilesStorageSize = testFileSize1 + testFileSize2;
 
-        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, false, apiToken);
+        // Get the total size when there are no tabular files
+        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedTextFilesStorageSize));
 
@@ -3670,20 +3671,31 @@ public void getDownloadSize() throws IOException {
         // Get the original tabular file size
         int tabularOriginalSize = Integer.parseInt(uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.filesize"));
 
-        // Get the size ignoring the original tabular file sizes
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, true, apiToken);
+        // Ensure tabular file is ingested
+        Thread.sleep(2000);
+
+        // Get the total size ignoring the original tabular file sizes
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Archival.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         int actualSizeIgnoringOriginalTabularSizes = Integer.parseInt(getDownloadSizeResponse.getBody().jsonPath().getString("data.storageSize"));
+
         // Assert that the size has been incremented with the last uploaded file
         assertTrue(actualSizeIgnoringOriginalTabularSizes > expectedTextFilesStorageSize);
 
-        // Get the size including the original tabular file sizes
-        int tabularProcessedSize = actualSizeIgnoringOriginalTabularSizes - expectedTextFilesStorageSize;
-        int expectedSizeIncludingOriginalTabularSizes = tabularOriginalSize + tabularProcessedSize + expectedTextFilesStorageSize;
+        // Get the total size including only original sizes and ignoring archival sizes for tabular files
+        int expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedTextFilesStorageSize;
+
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data.storageSize", equalTo(expectedSizeIncludingOnlyOriginalForTabular));
+
+        // Get the total size including both the original and archival tabular file sizes
+        int tabularArchivalSize = actualSizeIgnoringOriginalTabularSizes - expectedTextFilesStorageSize;
+        int expectedSizeIncludingAllSizes = tabularArchivalSize + tabularOriginalSize + expectedTextFilesStorageSize;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, false, apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
-                .body("data.storageSize", equalTo(expectedSizeIncludingOriginalTabularSizes));
+                .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index ecf26bd26ae..f9f3dc9be8d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3410,10 +3410,10 @@ static Response deaccessionDataset(Integer datasetId, String version, String api
                 .put("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
     }
 
-    static Response getDownloadSize(Integer datasetId, String version, boolean ignoreOriginalTabularSize, String apiToken) {
+    static Response getDownloadSize(Integer datasetId, String version, String mode, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .queryParam("ignoreOriginalTabularSize", ignoreOriginalTabularSize)
+                .queryParam("mode", mode)
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/downloadsize");
     }
 }

From a5c32bd1b11f4385926f9abc53578e6b48c05adc Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 10:53:45 +0100
Subject: [PATCH 479/922] Added: error case to getDownloadSize IT

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 189cf3a6f5a..ee3355096b8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3697,5 +3697,11 @@ public void getDownloadSize() throws IOException, InterruptedException {
         getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
+
+        // Get the total size sending invalid file download size mode
+        String invalidMode = "invalidMode";
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, invalidMode, apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", equalTo("Invalid mode: " + invalidMode));
     }
 }

From 131cd8f83473e9919e871723551eb441b6f27c3e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 11:22:44 +0100
Subject: [PATCH 480/922] Added: multiple tab files test case for
 getDownloadSize IT

---
 .../harvard/iq/dataverse/api/DatasetsIT.java   | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index ee3355096b8..829c19c6440 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3703,5 +3703,23 @@ public void getDownloadSize() throws IOException, InterruptedException {
         getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, invalidMode, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid mode: " + invalidMode));
+
+        // Upload second test tabular file (same source as before)
+        uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get the total size including only original sizes and ignoring archival sizes for tabular files
+        expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedSizeIncludingOnlyOriginalForTabular;
+
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data.storageSize", equalTo(expectedSizeIncludingOnlyOriginalForTabular));
+
+        // Get the total size including both the original and archival tabular file sizes
+        expectedSizeIncludingAllSizes = tabularArchivalSize + tabularOriginalSize + expectedSizeIncludingAllSizes;
+
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
     }
 }

From d1d5eed683dfc8d04cdf832d5c40d4947821b8c3 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <poikilotherm@users.noreply.github.com>
Date: Mon, 2 Oct 2023 12:34:33 +0200
Subject: [PATCH 481/922] style(oidc): apply language fixes by @pdurbin from
 review #9268

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 doc/release-notes/9268-8349-oidc-improvements.md | 15 ++++++++++++++-
 doc/sphinx-guides/source/installation/oidc.rst   | 10 +++++-----
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/doc/release-notes/9268-8349-oidc-improvements.md b/doc/release-notes/9268-8349-oidc-improvements.md
index cb0a9685c69..34e45a98101 100644
--- a/doc/release-notes/9268-8349-oidc-improvements.md
+++ b/doc/release-notes/9268-8349-oidc-improvements.md
@@ -5,7 +5,7 @@
 With this release it is possible to provision a single OIDC-based authentication provider
 by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
 
-If you are using an external OIDC provider component as identity management system and/or broker
+If you are using an external OIDC provider component as an identity management system and/or broker
 to other authentication providers such as Google, eduGain SAML and so on, this might make your
 life easier during instance setups and reconfiguration. You no longer need to generate the
 necessary JSON file.
@@ -26,3 +26,16 @@ The testing and development Keycloak realm has been updated with more users and
 The support for setting JVM options during testing has been improved for developers. You now may add the
 `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
 also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
+
+## New Configuration Options
+
+- dataverse.auth.oidc.enabled
+- dataverse.auth.oidc.client-id
+- dataverse.auth.oidc.client-secret
+- dataverse.auth.oidc.auth-server-url
+- dataverse.auth.oidc.pkce.enabled
+- dataverse.auth.oidc.pkce.method
+- dataverse.auth.oidc.title
+- dataverse.auth.oidc.subtitle
+- dataverse.auth.oidc.pkce.max-cache-size
+- dataverse.auth.oidc.pkce.max-cache-age
diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index e036e9c8470..4a0c77006e0 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -78,14 +78,14 @@ Enabling PKCE Security
 Many providers these days support or even require the usage of `PKCE <https://oauth.net/2/pkce/>`_ to safeguard against
 some attacks and enable public clients that cannot have a secure secret to still use OpenID Connect (or OAuth2).
 
-The Dataverse built OIDC client can be enabled to use PKCE and which method to use when creating the code challenge.
+The Dataverse-built OIDC client can be configured to use PKCE and the method to use when creating the code challenge can be specified.
 See also `this explanation of the flow <https://auth0.com/docs/get-started/authentication-and-authorization-flow/authorization-code-flow-with-proof-key-for-code-exchange-pkce>`_
 for details on how this works.
 
 As we are using the `Nimbus SDK <https://connect2id.com/products/nimbus-oauth-openid-connect-sdk>`_ as our client
-library, we support the standard ``PLAIN`` and ``S256`` code challenge methods. "SHA-256 method" is the default
+library, we support the standard ``PLAIN`` and ``S256`` (SHA-256) code challenge methods. "SHA-256 method" is the default
 as recommend in `RFC7636 <https://datatracker.ietf.org/doc/html/rfc7636#section-4.2>`_. If your provider needs some
-other method (unlikely), please open an issue.
+other method, please open an issue.
 
 The provisioning sections below contain in the example the parameters you may use to configure PKCE.
 
@@ -169,11 +169,11 @@ The following options are available:
     - N
     - ``OpenID Connect``
   * - ``dataverse.auth.oidc.pkce.max-cache-size``
-    - Tune the maximum size of all OIDC providers' verifier cache (= number of outstanding PKCE-enabled auth responses).
+    - Tune the maximum size of all OIDC providers' verifier cache (the number of outstanding PKCE-enabled auth responses).
     - N
     - 10000
   * - ``dataverse.auth.oidc.pkce.max-cache-age``
-    - Tune the maximum age of all OIDC providers' verifier cache entries. Default is 5 minutes, equivalent to lifetime
+    - Tune the maximum age, in seconds, of all OIDC providers' verifier cache entries. Default is 5 minutes, equivalent to lifetime
       of many OIDC access tokens.
     - N
     - 300
\ No newline at end of file

From 42d181216fd042d378768763209805f77e659149 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 2 Oct 2023 12:53:20 +0200
Subject: [PATCH 482/922] build(oidc): upgrade to Keycloak 22 in OIDC
 integration test #9268

---
 pom.xml                                                       | 2 +-
 .../oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pom.xml b/pom.xml
index 13a834e9a67..e70b723cad5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -609,7 +609,7 @@
         <dependency>
             <groupId>com.github.dasniko</groupId>
             <artifactId>testcontainers-keycloak</artifactId>
-            <version>2.5.0</version>
+            <version>3.0.0</version>
             <scope>test</scope>
         </dependency>
         <!--
diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
index e794bfa7d48..5968cf3eaeb 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -74,14 +74,14 @@ class OIDCAuthenticationProviderFactoryIT {
     
     // The realm JSON resides in conf/keycloak/test-realm.json and gets avail here using <testResources> in pom.xml
     @Container
-    static KeycloakContainer keycloakContainer = new KeycloakContainer("quay.io/keycloak/keycloak:21.0")
+    static KeycloakContainer keycloakContainer = new KeycloakContainer("quay.io/keycloak/keycloak:22.0")
         .withRealmImportFile("keycloak/test-realm.json")
         .withAdminUsername(adminUser)
         .withAdminPassword(adminPassword);
     
     // simple method to retrieve the issuer URL, referenced to by @JvmSetting annotations (do no delete)
     private static String getAuthUrl() {
-        return keycloakContainer.getAuthServerUrl() + "realms/" + realm;
+        return keycloakContainer.getAuthServerUrl() + "/realms/" + realm;
     }
     
     OIDCAuthProvider getProvider() throws Exception {

From 725bbb7c6c7c76a87f9892501b8050c24e704f8d Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 12:20:45 +0100
Subject: [PATCH 483/922] Changed: updated docs for /downloadsize endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 0f77aeba580..0cea70c04f1 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1797,13 +1797,17 @@ The fully expanded example above (without environment variables) looks like this
 The size of all files available for download will be returned. 
 If :draft is passed as versionId the token supplied must have permission to view unpublished drafts. A token is not required for published datasets. Also restricted files will be included in this total regardless of whether the user has access to download the restricted file(s).
 
-There is an optional query parameter ``ignoreOriginalTabularSize`` which, if set to true, the endpoint will return the download size ignoring the sizes of the original tabular files. Otherwise, both the original and the processed size will be included in the count for tabular files.
+There is an optional query parameter ``mode`` which applies a filter criteria to the operation. This parameter supports the following values:
+
+* ``All`` (Default): Includes both archival and original sizes for tabular files
+* ``Archival``: Includes only the archival size for tabular files
+* ``Original``: Includes only the original size for tabular files
 
 Usage example:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?ignoreOriginalTabularSize=true"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?mode=Archival"
 
 Submit a Dataset for Review
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

From cbf00d788ef27fdbe846328223d3fed9b00125bc Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 12:22:38 +0100
Subject: [PATCH 484/922] Changed: updated release notes for #9958

---
 ...958-dataset-api-downloadsize-ignore-tabular-size.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md b/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
index 73b27a1a581..2ede679b361 100644
--- a/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
+++ b/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
@@ -1,3 +1,9 @@
-Added a new optional query parameter "ignoreOriginalTabularSize" to the "getDownloadSize" API endpoint ("api/datasets/{identifier}/versions/{versionId}/downloadsize").
+Added a new optional query parameter "mode" to the "getDownloadSize" API endpoint ("api/datasets/{identifier}/versions/{versionId}/downloadsize").
 
-If set to true, the endpoint will return the download size ignoring the original tabular file sizes.
+This parameter applies a filter criteria to the operation and supports the following values:
+
+- All (Default): Includes both archival and original sizes for tabular files
+
+- Archival: Includes only the archival size for tabular files 
+
+- Original: Includes only the original size for tabular files

From 87c6515e3c22b25c66850714dfe17167b1202433 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 12:24:43 +0100
Subject: [PATCH 485/922] Added: sleep call to getDownloadSize IT to ensure tab
 file is ingested

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 829c19c6440..cab468fb1e9 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3708,6 +3708,9 @@ public void getDownloadSize() throws IOException, InterruptedException {
         uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
         uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
 
+        // Ensure tabular file is ingested
+        Thread.sleep(2000);
+
         // Get the total size including only original sizes and ignoring archival sizes for tabular files
         expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedSizeIncludingOnlyOriginalForTabular;
 

From 38cabc1767fa61e59c16aefcabc03d5514006d7e Mon Sep 17 00:00:00 2001
From: Guillermo Portas <hey@gportas.me>
Date: Mon, 2 Oct 2023 12:58:36 +0100
Subject: [PATCH 486/922] Changed: getVersionFiles docs suggestions applied

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 01a681cfb6a..4a84cc17d16 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1020,7 +1020,7 @@ Usage example:
 
 Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
 
-By default, deaccessioned dataset versions are not supported by this endpoint and will be ignored in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a not found error if the version is deaccessioned and you do not enable the option described below.
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
 
 If you want to consider deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
 

From 5c7830c9022d72cb2bdd248981f23fc4d29fd1d0 Mon Sep 17 00:00:00 2001
From: Guillermo Portas <hey@gportas.me>
Date: Mon, 2 Oct 2023 12:59:48 +0100
Subject: [PATCH 487/922] Changed: getVersionFiles docs suggestions applied (2)

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 4a84cc17d16..9459440608b 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1022,7 +1022,7 @@ Please note that both filtering and ordering criteria values are case sensitive
 
 By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
 
-If you want to consider deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
+If you want to consider deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
 
 Usage example:
 

From 02003f12853ed441c30c1e3a1e51e38824b3defb Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 13:14:57 +0100
Subject: [PATCH 488/922] Added: clarification to Deaccession Dataset API docs
 about calling the endpoint multiple times for the same dataset version

---
 doc/sphinx-guides/source/api/native-api.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 01a681cfb6a..4bca19d078d 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1385,6 +1385,8 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -d '{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
 
+.. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be public since it is already deaccessioned.
+
 Set Citation Date Field Type for a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From bb0ba9abfd934415b6a8a2c24da3cab1f279b291 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 2 Oct 2023 15:16:50 +0200
Subject: [PATCH 489/922] doc(oidc): move example JSON to downloadable file
 #9268

---
 .../files/root/auth-providers/oidc.json         |  8 ++++++++
 doc/sphinx-guides/source/installation/oidc.rst  | 17 +++++------------
 2 files changed, 13 insertions(+), 12 deletions(-)
 create mode 100644 doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json

diff --git a/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json
new file mode 100644
index 00000000000..9df38988a25
--- /dev/null
+++ b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json
@@ -0,0 +1,8 @@
+{
+    "id":"<a unique id>",
+    "factoryAlias":"oidc",
+    "title":"<a title - shown in UI>",
+    "subtitle":"<a subtitle - currently unused in UI>",
+    "factoryData":"type: oidc | issuer: <issuer url> | clientId: <client id> | clientSecret: <client secret> | pkceEnabled: <true/false> | pkceMethod: <PLAIN/S256/...>",
+    "enabled":true
+}
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index 4a0c77006e0..50cb494330b 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -95,18 +95,11 @@ Provision via REST API
 Note: you may omit the PKCE related settings from ``factoryData`` below if you don't plan on using PKCE - default is
 disabled.
 
-Please create a ``my-oidc-provider.json`` file like this, replacing every ``<...>`` with your values:
-
-.. code-block:: json
-
-    {
-        "id":"<a unique id>",
-        "factoryAlias":"oidc",
-        "title":"<a title - shown in UI>",
-        "subtitle":"<a subtitle - currently unused in UI>",
-        "factoryData":"type: oidc | issuer: <issuer url> | clientId: <client id> | clientSecret: <client secret> | pkceEnabled: <true/false> | pkceMethod: <PLAIN/S256/...>",
-        "enabled":true
-    }
+Please create a :download:`my-oidc-provider.json <../_static/installation/files/root/auth-providers/oidc.json>` file, replacing every ``<...>`` with your values:
+
+.. literalinclude:: /_static/installation/files/root/auth-providers/oidc.json
+    :name: oidc-provider-example
+    :language: json
 
 Now load the configuration into your Dataverse installation using the same API as with :doc:`oauth2`:
 

From ce01176cc6e760ba37c2f1534a8ad551daa2e0cc Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Mon, 2 Oct 2023 15:29:30 +0200
Subject: [PATCH 490/922] doc(oidc): extend a little on when to use which
 provisioning way #9268

Following a suggestion by @pdurbin here from code review.
---
 doc/sphinx-guides/source/installation/oidc.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index 50cb494330b..4434573bea2 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -89,6 +89,20 @@ other method, please open an issue.
 
 The provisioning sections below contain in the example the parameters you may use to configure PKCE.
 
+Provision a provider
+--------------------
+
+Depending on your use case, you can choose different ways to setup one or multiple OIDC identity providers.
+
+Using MicroProfile Config has the advantage of being consistent and does not require additional calls to the API.
+It can only configure one provider though, yet you can mix with other provider definitions via API.
+
+Using the REST API has the advantage of provisioning multiple, different OIDC providers.
+Depending on your use case, it has the drawback of needing additional API calls.
+
+If you only need one single provider in your installation and it is using OIDC, use the MPCONFIG option, as it requires
+fewer extra steps and allows you to keep more configuration in a single source.
+
 Provision via REST API
 ^^^^^^^^^^^^^^^^^^^^^^
 

From f9fba208541d752ac40d406e9a2fa835a9e256a0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 2 Oct 2023 09:58:18 -0400
Subject: [PATCH 491/922] clarify wording per code review #9589

---
 doc/sphinx-guides/source/admin/external-tools.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index 68b5f493d49..346ca0b15ee 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -115,7 +115,7 @@ Dataset level explore tools allow the user to explore all the files in a dataset
 Dataset Level Configure Tools
 +++++++++++++++++++++++++++++
 
-Dataset level configure tools can be launched by users who have edit access to the dataset and are found under the "Edit Dataset" menu.
+Dataset level configure tools can be launched by users who have edit access to the dataset. These tools are found under the "Edit Dataset" menu.
 
 Writing Your Own External Tool
 ------------------------------

From 7540fbf653f1c80e6fab965b36e6bc897bac2fac Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 2 Oct 2023 10:30:44 -0400
Subject: [PATCH 492/922] call configure method rather than explore #9589

---
 src/main/webapp/dataset.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 5bb6f66ed3e..1b1d77d3057 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1008,7 +1008,7 @@
                             <h:outputFormat value="#{DatasetPage.datasetConfigureTool.description}" escape="false"/>
                         </p>
                         <div class="button-block">
-                            <p:commandLink styleClass="btn btn-default" action="#{DatasetPage.explore(DatasetPage.datasetConfigureTool)}" oncomplete="PF('configureToolDialog').hide();">
+                            <p:commandLink styleClass="btn btn-default" action="#{DatasetPage.configure(DatasetPage.datasetConfigureTool)}" oncomplete="PF('configureToolDialog').hide();">
                                 <h:outputFormat value="#{bundle['continue']}"/>
                             </p:commandLink>
                             <button class="btn btn-link" onclick="PF('configureToolDialog').hide();" type="button">

From 244ade84624970c0040b4ec29c7250ab550cdeda Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 2 Oct 2023 11:30:53 -0400
Subject: [PATCH 493/922] tiny doc fixes #9852

---
 doc/sphinx-guides/source/api/native-api.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 78576e8bbe1..377ca4017f6 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1022,7 +1022,7 @@ Please note that both filtering and ordering criteria values are case sensitive
 
 By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
 
-If you want to consider deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
 
 Usage example:
 
@@ -1060,7 +1060,7 @@ The fully expanded example above (without environment variables) looks like this
 
 By default, deaccessioned dataset versions are not supported by this endpoint and will be ignored in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a not found error if the version is deaccessioned and you do not enable the option described below.
 
-If you want to consider deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
+If you want to include deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
 
 Usage example:
 
@@ -1385,7 +1385,7 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -d '{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
 
-.. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be public since it is already deaccessioned.
+.. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be published since it is already deaccessioned.
 
 Set Citation Date Field Type for a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 649ea13e9dcf80b941de0cca209d389fd810e352 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 2 Oct 2023 18:24:33 +0100
Subject: [PATCH 494/922] Changed: using POST method for deaccessionDataset API
 endpoint and string bundle for error messages (IT extended)

---
 doc/sphinx-guides/source/api/native-api.rst   |  4 +-
 .../harvard/iq/dataverse/api/Datasets.java    |  6 +--
 src/main/java/propertyFiles/Bundle.properties |  3 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 48 +++++++++++++++----
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  9 ++--
 5 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 78576e8bbe1..f494415e731 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1377,13 +1377,13 @@ Given a version of a dataset, updates its status to deaccessioned.
   export VERSIONID=1.0
   export JSON='{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession" -d "$JSON"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession" -d "$JSON"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -d '{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -d '{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
 
 .. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be public since it is already deaccessioned.
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 62d87b198fe..e334116958d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3924,12 +3924,12 @@ public Response getDatasetVersionCitation(@Context ContainerRequestContext crc,
                 getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getCitation(true, false)), getRequestUser(crc));
     }
 
-    @PUT
+    @POST
     @AuthRequired
     @Path("{id}/versions/{versionId}/deaccession")
     public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, String jsonBody, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         if (DS_VERSION_DRAFT.equals(versionId) || DS_VERSION_LATEST.equals(versionId)) {
-            return badRequest("Only " + DS_VERSION_LATEST_PUBLISHED + " or a specific version can be deaccessioned");
+            return badRequest(BundleUtil.getStringFromBundle("datasets.api.deaccessionDataset.invalid.version.identifier.error", List.of(DS_VERSION_LATEST_PUBLISHED)));
         }
         return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, false);
@@ -3941,7 +3941,7 @@ public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathPa
                     try {
                         datasetVersion.setArchiveNote(deaccessionForwardURL);
                     } catch (IllegalArgumentException iae) {
-                        return error(Response.Status.BAD_REQUEST, "Invalid deaccession forward URL: " + iae.getMessage());
+                        return badRequest(BundleUtil.getStringFromBundle("datasets.api.deaccessionDataset.invalid.forward.url", List.of(iae.getMessage())));
                     }
                 }
                 execCommand(new DeaccessionDatasetVersionCommand(req, datasetVersion, false));
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 997f0470cc3..6e7ed55a768 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2623,7 +2623,8 @@ datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymi
 datasets.api.creationdate=Date Created
 datasets.api.modificationdate=Last Modified Date
 datasets.api.curationstatus=Curation Status
-
+datasets.api.deaccessionDataset.invalid.version.identifier.error=Only {0} or a specific version can be deaccessioned
+datasets.api.deaccessionDataset.invalid.forward.url=Invalid deaccession forward URL: {0}
 
 #Dataverses.java
 dataverses.api.update.default.contributor.role.failure.role.not.found=Role {0} not found.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 5c1eb66b63d..cfb430e6995 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3,6 +3,7 @@
 import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
 import io.restassured.RestAssured;
 
+import static edu.harvard.iq.dataverse.DatasetVersion.ARCHIVE_NOTE_MAX_LENGTH;
 import static edu.harvard.iq.dataverse.api.ApiConstants.*;
 import static io.restassured.RestAssured.given;
 
@@ -15,6 +16,7 @@
 import java.util.*;
 import java.util.logging.Logger;
 
+import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
@@ -3608,26 +3610,54 @@ public void deaccessionDataset() {
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
-        // Test that draft and latest version constants are not allowed
-        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_DRAFT, apiToken);
-        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST, apiToken);
-        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+        String testDeaccessionReason = "Test deaccession reason.";
+        String testDeaccessionForwardURL = "http://demo.dataverse.org";
+
+        // Test that draft and latest version constants are not allowed and a bad request error is received
+        String expectedInvalidVersionIdentifierError = BundleUtil.getStringFromBundle("datasets.api.deaccessionDataset.invalid.version.identifier.error", List.of(DS_VERSION_LATEST_PUBLISHED));
+
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_DRAFT, testDeaccessionReason, testDeaccessionForwardURL, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", equalTo(expectedInvalidVersionIdentifierError));
+
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST, testDeaccessionReason, testDeaccessionForwardURL, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", equalTo(expectedInvalidVersionIdentifierError));
 
         // Test that a not found error occurs when there is no published version available
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, testDeaccessionReason, testDeaccessionForwardURL, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
 
-        // Test that the dataset is successfully deaccessioned when published
+        // Publish test dataset
         Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
         publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
         Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
         publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
+
+        // Test that a bad request error is received when the forward URL exceeds ARCHIVE_NOTE_MAX_LENGTH
+        String testInvalidDeaccessionForwardURL = RandomStringUtils.randomAlphabetic(ARCHIVE_NOTE_MAX_LENGTH + 1);
+
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, testDeaccessionReason, testInvalidDeaccessionForwardURL, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", containsString(testInvalidDeaccessionForwardURL));
+
+        // Test that the dataset is successfully deaccessioned when published and valid deaccession params are sent
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, testDeaccessionReason, testDeaccessionForwardURL, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // Test that a not found error occurs when the only published version has already been deaccessioned
-        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, testDeaccessionReason, testDeaccessionForwardURL, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // Test that a dataset can be deaccessioned without forward URL
+        createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, testDeaccessionReason, null, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 8c6a2d6e75d..9a5ef76a5ff 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3400,13 +3400,16 @@ static Response getHasBeenDeleted(String dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/hasBeenDeleted");
     }
 
-    static Response deaccessionDataset(Integer datasetId, String version, String apiToken) {
+    static Response deaccessionDataset(Integer datasetId, String version, String deaccessionReason, String deaccessionForwardURL, String apiToken) {
         JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        jsonObjectBuilder.add("deaccessionReason", "Test deaccession.");
+        jsonObjectBuilder.add("deaccessionReason", deaccessionReason);
+        if (deaccessionForwardURL != null) {
+            jsonObjectBuilder.add("deaccessionForwardURL", deaccessionForwardURL);
+        }
         String jsonString = jsonObjectBuilder.build().toString();
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .body(jsonString)
-                .put("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
+                .post("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
     }
 }

From cb1f3e65fc06302d37dae280714ffb0bce764b39 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 2 Oct 2023 13:26:46 -0400
Subject: [PATCH 495/922] #9507 prelim check in

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 19 ++++++++++++++++++-
 src/main/java/propertyFiles/Bundle.properties |  1 +
 src/main/webapp/dataset.xhtml                 |  8 ++++++++
 3 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index f99c10b2b79..3e5b2d78f68 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -61,6 +61,7 @@
 import edu.harvard.iq.dataverse.util.WebloaderUtil;
 import edu.harvard.iq.dataverse.validation.URLValidator;
 import edu.harvard.iq.dataverse.workflows.WorkflowComment;
+import edu.harvard.iq.dataverse.Dataverse;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -3423,7 +3424,23 @@ private Boolean saveLink(Dataverse dataverse){
         }
         return retVal;
     }
-
+        
+    private String alreadyLinkedDataverses = null;
+    
+    public String getAlreadyLinkedDataverses(){
+        if (alreadyLinkedDataverses != null) {           
+            return alreadyLinkedDataverses;
+        }
+        List<Dataverse> dataverseList = dataverseService.findDataversesThatLinkToThisDatasetId(dataset.getId());
+        for (Dataverse dv: dataverseList){
+            if (alreadyLinkedDataverses == null){
+                alreadyLinkedDataverses = dv.getCurrentName();
+            } else {
+                alreadyLinkedDataverses = alreadyLinkedDataverses + ", " + dv.getCurrentName();
+            }
+        }
+        return alreadyLinkedDataverses;
+    }
 
     public List<Dataverse> completeLinkingDataverse(String query) {
         dataset = datasetService.find(dataset.getId());
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 55ffcd2f57d..2f5b0f64e6c 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2494,6 +2494,7 @@ dataset.registered=DatasetRegistered
 dataset.registered.msg=Your dataset is now registered.
 dataset.notlinked=DatasetNotLinked
 dataset.notlinked.msg=There was a problem linking this dataset to yours:
+dataset.linking.popop.already.linked.note=Note: the dataset is already linked to the following dataverses: 
 datasetversion.archive.success=Archival copy of Version successfully submitted
 datasetversion.archive.failure=Error in submitting an archival copy 
 datasetversion.update.failure=Dataset Version Update failed. Changes are still in the DRAFT version.
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index a9f1ff339f0..6cc1f5cc45c 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1598,6 +1598,14 @@
                                 </div>
                             </div>
                         </div>
+                        <div>
+                            <p:outputLabel rendered="#{!empty DatasetPage.alreadyLinkedDataverses }" >
+                         #{bundle.dataset.linking.popop.already.linked.note}
+                            </p:outputLabel>
+                            <h:outputText rendered="#{!empty DatasetPage.alreadyLinkedDataverses }">
+                            #{DatasetPage.alreadyLinkedDataverses}
+                            </h:outputText>
+                        </div>
                         <div class="button-block">
                             <p:commandButton id="saveLinkButton" styleClass="btn btn-default"    
                                              update="linkNameContent @([id$=Messages])" 

From 9ff707d11cc1c955b8a4d3e9e891b1788e83188d Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 2 Oct 2023 14:07:19 -0400
Subject: [PATCH 496/922] #9507 fix bundle/format

---
 src/main/java/propertyFiles/Bundle.properties | 2 +-
 src/main/webapp/dataset.xhtml                 | 8 ++------
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 2f5b0f64e6c..e14374780c8 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2494,7 +2494,7 @@ dataset.registered=DatasetRegistered
 dataset.registered.msg=Your dataset is now registered.
 dataset.notlinked=DatasetNotLinked
 dataset.notlinked.msg=There was a problem linking this dataset to yours:
-dataset.linking.popop.already.linked.note=Note: the dataset is already linked to the following dataverses: 
+dataset.linking.popop.already.linked.note=Note: the dataset is already linked to the following dataverses:  
 datasetversion.archive.success=Archival copy of Version successfully submitted
 datasetversion.archive.failure=Error in submitting an archival copy 
 datasetversion.update.failure=Dataset Version Update failed. Changes are still in the DRAFT version.
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 6cc1f5cc45c..eccec7bf6a3 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1599,12 +1599,8 @@
                             </div>
                         </div>
                         <div>
-                            <p:outputLabel rendered="#{!empty DatasetPage.alreadyLinkedDataverses }" >
-                         #{bundle.dataset.linking.popop.already.linked.note}
-                            </p:outputLabel>
-                            <h:outputText rendered="#{!empty DatasetPage.alreadyLinkedDataverses }">
-                            #{DatasetPage.alreadyLinkedDataverses}
-                            </h:outputText>
+                            <h:outputLabel rendered="#{!empty DatasetPage.alreadyLinkedDataverses }" value="#{bundle['dataset.linking.popop.already.linked.note']} "/>
+                            <h:outputText rendered="#{!empty DatasetPage.alreadyLinkedDataverses }" value="#{DatasetPage.alreadyLinkedDataverses}"/>
                         </div>
                         <div class="button-block">
                             <p:commandButton id="saveLinkButton" styleClass="btn btn-default"    

From efe0052f7117e0c57fa122c98f14bb51ba1b8e3a Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 2 Oct 2023 16:40:39 -0400
Subject: [PATCH 497/922] #9507 update ui; add refresh logic

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 2 +-
 src/main/java/propertyFiles/Bundle.properties           | 2 +-
 src/main/webapp/dataset.xhtml                           | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 3e5b2d78f68..d53db206afa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3388,7 +3388,7 @@ public void saveLinkingDataverses(ActionEvent evt) {
             FacesMessage message = new FacesMessage(FacesMessage.SEVERITY_INFO, BundleUtil.getStringFromBundle("dataset.notlinked"), linkingDataverseErrorMessage);
             FacesContext.getCurrentInstance().addMessage(null, message);
         }
-
+        alreadyLinkedDataverses = null; //force update to list of linked dataverses
     }
 
     private String linkingDataverseErrorMessage = "";
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index b5992a35a3f..0c96192ee48 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2497,7 +2497,7 @@ dataset.registered=DatasetRegistered
 dataset.registered.msg=Your dataset is now registered.
 dataset.notlinked=DatasetNotLinked
 dataset.notlinked.msg=There was a problem linking this dataset to yours:
-dataset.linking.popop.already.linked.note=Note: the dataset is already linked to the following dataverses:  
+dataset.linking.popop.already.linked.note=Note: This dataset is already linked to the following dataverse(s):  
 datasetversion.archive.success=Archival copy of Version successfully submitted
 datasetversion.archive.failure=Error in submitting an archival copy 
 datasetversion.update.failure=Dataset Version Update failed. Changes are still in the DRAFT version.
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index eccec7bf6a3..58ee9b6dbdc 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -1599,8 +1599,10 @@
                             </div>
                         </div>
                         <div>
-                            <h:outputLabel rendered="#{!empty DatasetPage.alreadyLinkedDataverses }" value="#{bundle['dataset.linking.popop.already.linked.note']} "/>
+                            <p:fragment rendered="#{!empty DatasetPage.alreadyLinkedDataverses}"> 
+                            <h:outputLabel  value="#{bundle['dataset.linking.popop.already.linked.note']}"/>&#160;
                             <h:outputText rendered="#{!empty DatasetPage.alreadyLinkedDataverses }" value="#{DatasetPage.alreadyLinkedDataverses}"/>
+                            </p:fragment>
                         </div>
                         <div class="button-block">
                             <p:commandButton id="saveLinkButton" styleClass="btn btn-default"    

From 274f0e989de5bcac26746c1e31193bd985616495 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 3 Oct 2023 09:42:44 +0200
Subject: [PATCH 498/922] style(oidc): apply title case to docs as requested by
 @pdurbin #9268

---
 doc/sphinx-guides/source/installation/oidc.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index 4434573bea2..6b405724450 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -26,7 +26,7 @@ You can also either host an OpenID Connect identity management on your own or us
 - `Keycloak <https://www.keycloak.org>`_ is an open source solution for an IDM/IAM
 - `Unity IDM <https://www.unity-idm.eu>`_ is another open source IDM/IAM solution
 
-Other use cases and combinations
+Other Use Cases and Combinations
 --------------------------------
 
 - Using your custom identity management solution might be a workaround when you seek for LDAP support, but
@@ -41,7 +41,7 @@ Other use cases and combinations
 - In the future, OpenID Connect might become a successor to the large scale R&E SAML federations we have nowadays.
   See also `OpenID Connect Federation Standard <https://openid.net/specs/openid-connect-federation-1_0.html>`_ (in development)
 
-How to use
+How to Use
 ----------
 
 Just like with :doc:`oauth2` you need to obtain a *Client ID* and a *Client Secret* from your provider(s).
@@ -89,7 +89,7 @@ other method, please open an issue.
 
 The provisioning sections below contain in the example the parameters you may use to configure PKCE.
 
-Provision a provider
+Provision a Provider
 --------------------
 
 Depending on your use case, you can choose different ways to setup one or multiple OIDC identity providers.

From a2b9d56f7309d4d0c0e3700d6c1ae6bf34de82f5 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 3 Oct 2023 10:44:45 +0200
Subject: [PATCH 499/922] doc(oidc,settings): align terminology to JVM options
 and explain some more about MPCONFIG

---
 .../source/installation/config.rst            |  5 +++++
 .../source/installation/oidc.rst              | 20 +++++++++++--------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index da1f86e4e9b..c00e673384c 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1699,6 +1699,11 @@ When changing values these values with ``asadmin``, you'll need to delete the ol
 
 It's also possible to change these values by stopping Payara, editing ``payara6/glassfish/domains/domain1/config/domain.xml``, and restarting Payara.
 
+In addition, JVM options enabled for "MicroProfile Config" (see docs of any option), can be used with any
+`supported MicroProfile Config API source`_ to provide their values. The most notable source are environment variables;
+many examples are given in detail documentation of enabled options.
+
+
 .. _dataverse.fqdn:
 
 dataverse.fqdn
diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index 6b405724450..e8154f0edee 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -94,14 +94,14 @@ Provision a Provider
 
 Depending on your use case, you can choose different ways to setup one or multiple OIDC identity providers.
 
-Using MicroProfile Config has the advantage of being consistent and does not require additional calls to the API.
+Using :ref:`jvm-options` has the advantage of being consistent and does not require additional calls to the API.
 It can only configure one provider though, yet you can mix with other provider definitions via API.
 
 Using the REST API has the advantage of provisioning multiple, different OIDC providers.
 Depending on your use case, it has the drawback of needing additional API calls.
 
-If you only need one single provider in your installation and it is using OIDC, use the MPCONFIG option, as it requires
-fewer extra steps and allows you to keep more configuration in a single source.
+If you only need one single provider in your installation and it is using OIDC, use the JVM options, as it
+requires fewer extra steps and allows you to keep more configuration in a single source.
 
 Provision via REST API
 ^^^^^^^^^^^^^^^^^^^^^^
@@ -125,12 +125,16 @@ section of the User Guide.
 
 .. _oidc-mpconfig:
 
-Provision via MPCONFIG
-^^^^^^^^^^^^^^^^^^^^^^
+Provision via JVM Options
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A single provider may be provisioned using :ref:`jvm-options`.
+It may be accompanied by more providers configured via REST API.
+Note that this provider will only be deployed at startup time and (currently) cannot be reconfigured without a restart.
 
-In case you only require a single OIDC provider, you can also provision an OIDC based provider using MicroProfile Config
-sources. Note that this provider will only be deployed at startup time and (currently) cannot be reconfigured without
-a restart.
+All options below may be set via *MicroProfile Config API* sources. Examples: use environment variable
+``DATAVERSE_AUTH_OIDC_ENABLED`` for the ``dataverse.auth.oidc.enabled`` option or ``DATAVERSE_AUTH_OIDC_CLIENT_ID``
+for the ``dataverse.auth.oidc.client-id`` option.
 
 The following options are available:
 

From cee91f354ad52d065bc41fd754d15f2c1835cfc8 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <poikilotherm@users.noreply.github.com>
Date: Tue, 3 Oct 2023 10:47:58 +0200
Subject: [PATCH 500/922] doc(oidc): specify explanation of PKCE method
 parameter #9268

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 doc/sphinx-guides/source/installation/oidc.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/installation/oidc.rst b/doc/sphinx-guides/source/installation/oidc.rst
index e8154f0edee..34710f260e8 100644
--- a/doc/sphinx-guides/source/installation/oidc.rst
+++ b/doc/sphinx-guides/source/installation/oidc.rst
@@ -168,7 +168,7 @@ The following options are available:
     - N
     - ``false``
   * - ``dataverse.auth.oidc.pkce.method``
-    - Set code challenge method. Default equals best practice.
+    - Set code challenge method. The default value is the current best practice in the literature.
     - N
     - ``S256``
   * - ``dataverse.auth.oidc.title``

From 4561e837070d1e45b05c184ad843aab915b36a67 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 3 Oct 2023 09:49:13 +0100
Subject: [PATCH 501/922] Refactor: simpler where condition for
 getOriginalTabularFilesSize query

---
 .../harvard/iq/dataverse/DatasetVersionFilesServiceBean.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 66e0ec5b5fe..f957f7473dd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -269,7 +269,7 @@ private long getOriginalTabularFilesSize(DatasetVersion datasetVersion) {
                 .from(fileMetadata)
                 .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
                 .from(dataTable)
-                .where(fileMetadata.dataFile.dataTables.isNotEmpty().and(dataTable.dataFile.eq(fileMetadata.dataFile)))
+                .where(dataTable.dataFile.eq(fileMetadata.dataFile))
                 .select(dataTable.originalFileSize.sum()).fetchFirst();
         return (result == null) ? 0 : result;
     }

From 47da5a4508b2c8d2d38531ad370da81fac1bce7b Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 3 Oct 2023 09:56:04 +0100
Subject: [PATCH 502/922] Changed: using the known size of the tab file in IT
 instead of obtaining it from response

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index cab468fb1e9..b3c6535b493 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3668,8 +3668,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
         uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Get the original tabular file size
-        int tabularOriginalSize = Integer.parseInt(uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.filesize"));
+        int tabularOriginalSize = 157;
 
         // Ensure tabular file is ingested
         Thread.sleep(2000);

From 379022ce5bfde3fefd204a9da7ca9d33ea7ed94d Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <poikilotherm@users.noreply.github.com>
Date: Tue, 3 Oct 2023 11:01:12 +0200
Subject: [PATCH 503/922] doc(oidc,test): remove group column for predefined
 Keycloak users

As the OIDC provider does not yet provide proper support mapping from OIDC groups to Dataverse groups, remove this from the docs for now. Re-add when #9969 is (being) resolved.

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 doc/sphinx-guides/source/developers/remote-users.rst | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst
index a07f104d1c8..484abe9ccf0 100755
--- a/doc/sphinx-guides/source/developers/remote-users.rst
+++ b/doc/sphinx-guides/source/developers/remote-users.rst
@@ -61,19 +61,14 @@ You should be able to log into Keycloak with the one of the following credential
 
   * - Username
     - Password
-    - Group
   * - admin
     - admin
-    - admins
   * - curator
     - curator
-    - curators
   * - user
     - user
-    - members
   * - affiliate
     - affiliate
-    - \-
 
 In case you want to stop and remove the Keycloak container, just run the other available bash script:
 

From 91db30212d7f3b7c384ceeb1f04c9fcb9f12b808 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 3 Oct 2023 10:12:56 +0100
Subject: [PATCH 504/922] Fixed: failing IT due to missing params when calling
 deaccessionDataset

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index cfb430e6995..b23852e8221 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3499,7 +3499,7 @@ public void getVersionFiles() throws IOException {
         Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
         publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // includeDeaccessioned false
@@ -3581,7 +3581,7 @@ public void getVersionFileCounts() throws IOException {
         Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
         publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, apiToken);
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken);
         deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         // includeDeaccessioned false

From 982582bfc6e9f15d19cc13ecb22dc0e1dcfd93c8 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 3 Oct 2023 09:59:45 -0400
Subject: [PATCH 505/922] #9507 update bundle dataverse to collection

---
 src/main/java/propertyFiles/Bundle.properties | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 0c96192ee48..c9b212ffcf1 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -874,23 +874,23 @@ dataverse.publish.header=Publish Dataverse
 dataverse.nopublished=No Published Dataverses
 dataverse.nopublished.tip=In order to use this feature you must have at least one published dataverse.
 dataverse.contact=Email Dataverse Contact
-dataverse.link=Link Dataverse
-dataverse.link.btn.tip=Link to Your Dataverse
-dataverse.link.yourDataverses=Your Dataverse
-dataverse.link.yourDataverses.inputPlaceholder=Enter Dataverse Name
-dataverse.link.save=Save Linked Dataverse
-dataverse.link.dataverse.choose=Choose which of your dataverses you would like to link this dataverse to.
-dataverse.link.dataset.choose=Enter the name of the dataverse you would like to link this dataset to. If you need to remove this link in the future, please contact {0}.
-dataverse.link.dataset.none=No linkable dataverses available.
-dataverse.link.no.choice=You have one dataverse you can add linked dataverses and datasets in.
-dataverse.link.no.linkable=To be able to link a dataverse or dataset, you need to have your own dataverse. Create a dataverse to get started.
-dataverse.link.no.linkable.remaining=You have already linked all of your eligible dataverses.
+dataverse.link=Link Collection
+dataverse.link.btn.tip=Link to Your Collection
+dataverse.link.yourDataverses=Your Collection
+dataverse.link.yourDataverses.inputPlaceholder=Enter Collection Name
+dataverse.link.save=Save Linked collection
+dataverse.link.dataverse.choose=Choose which of your collection you would like to link this collection to.
+dataverse.link.dataset.choose=Enter the name of the collection you would like to link this dataset to. If you need to remove this link in the future, please contact {0}.
+dataverse.link.dataset.none=No linkable collections available.
+dataverse.link.no.choice=You have one collection you can add linked collection and datasets in.
+dataverse.link.no.linkable=To be able to link a collection or dataset, you need to have your own collection. Create a collection to get started.
+dataverse.link.no.linkable.remaining=You have already linked all of your eligible collections.
 dataverse.savedsearch.link=Link Search
 dataverse.savedsearch.searchquery=Search
 dataverse.savedsearch.filterQueries=Facets
 dataverse.savedsearch.save=Save Linked Search
-dataverse.savedsearch.dataverse.choose=Choose which of your dataverses you would like to link this search to.
-dataverse.savedsearch.no.choice=You have one dataverse to which you may add a saved search.
+dataverse.savedsearch.dataverse.choose=Choose which of your collection you would like to link this search to.
+dataverse.savedsearch.no.choice=You have one collection to which you may add a saved search.
 # Bundle file editors, please note that "dataverse.savedsearch.save.success" is used in a unit test
 dataverse.saved.search.success=The saved search has been successfully linked to {0}.
 dataverse.saved.search.failure=The saved search was not able to be linked.
@@ -2497,7 +2497,7 @@ dataset.registered=DatasetRegistered
 dataset.registered.msg=Your dataset is now registered.
 dataset.notlinked=DatasetNotLinked
 dataset.notlinked.msg=There was a problem linking this dataset to yours:
-dataset.linking.popop.already.linked.note=Note: This dataset is already linked to the following dataverse(s):  
+dataset.linking.popop.already.linked.note=Note: This dataset is already linked to the following collection(s):  
 datasetversion.archive.success=Archival copy of Version successfully submitted
 datasetversion.archive.failure=Error in submitting an archival copy 
 datasetversion.update.failure=Dataset Version Update failed. Changes are still in the DRAFT version.

From 47369ffe8b97d423a8ac63e3b8623d45fc82e000 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 3 Oct 2023 10:31:48 -0400
Subject: [PATCH 506/922] #9507 unneeded import

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 7713558852c..e9109c2b82b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -61,7 +61,6 @@
 import edu.harvard.iq.dataverse.util.WebloaderUtil;
 import edu.harvard.iq.dataverse.validation.URLValidator;
 import edu.harvard.iq.dataverse.workflows.WorkflowComment;
-import edu.harvard.iq.dataverse.Dataverse;
 
 import java.io.File;
 import java.io.FileOutputStream;

From e9833e83860ffbdab4db8168a77552c34445997c Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 3 Oct 2023 10:36:53 -0400
Subject: [PATCH 507/922] document new location for jacoco unit test file #9268

---
 doc/release-notes/9268-8349-oidc-improvements.md | 2 ++
 doc/sphinx-guides/source/developers/tools.rst    | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/9268-8349-oidc-improvements.md b/doc/release-notes/9268-8349-oidc-improvements.md
index 34e45a98101..ddfc13e603c 100644
--- a/doc/release-notes/9268-8349-oidc-improvements.md
+++ b/doc/release-notes/9268-8349-oidc-improvements.md
@@ -27,6 +27,8 @@ The support for setting JVM options during testing has been improved for develop
 `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
 also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
 
+As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
+
 ## New Configuration Options
 
 - dataverse.auth.oidc.enabled
diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst
index a21becd14cf..d45f6c43846 100755
--- a/doc/sphinx-guides/source/developers/tools.rst
+++ b/doc/sphinx-guides/source/developers/tools.rst
@@ -96,7 +96,7 @@ Download SonarQube from https://www.sonarqube.org and start look in the `bin` di
     -Dsonar.test.exclusions='src/test/**,src/main/webapp/resources/**' \
     -Dsonar.issuesReport.html.enable=true \
     -Dsonar.issuesReport.html.location='sonar-issues-report.html' \
-    -Dsonar.jacoco.reportPath=target/jacoco.exec
+    -Dsonar.jacoco.reportPath=target/coverage-reports/jacoco-unit.exec
 
 Once the analysis is complete, you should be able to access http://localhost:9000/dashboard?id=edu.harvard.iq%3Adataverse to see the report. To learn about resource leaks, for example, click on "Bugs", the "Tag", then "leak" or "Rule", then "Resources should be closed".
 

From 794045fbc0f43f0cebccc9ebb4f4b234a6c2215a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 3 Oct 2023 17:03:38 -0400
Subject: [PATCH 508/922] update auth checks and err handling

---
 .../harvard/iq/dataverse/api/Datasets.java    | 41 +++++++++++++++----
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 110bfcc1553..25839544ce9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -65,6 +65,7 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.S3AccessIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
@@ -3429,8 +3430,12 @@ public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathPar
         // -------------------------------------
         // (1) Get the user from the ContainerRequestContext
         // -------------------------------------
-        User authUser;
-        authUser = getRequestUser(crc);
+        AuthenticatedUser authUser;
+        try {
+            authUser = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse e) {
+            return e.getResponse();
+        }
 
         // -------------------------------------
         // (2) Get the Dataset Id
@@ -3442,14 +3447,32 @@ public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathPar
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
-        
-        JsonObject params = JsonUtil.getJsonObject(jsonBody);
-        String principal = params.getString("principal");
 
-        // Async Call
-        globusService.givePermission("identity", principal, "rw", dataset);
-
-        return ok("Permission Granted");
+        if(!GlobusOverlayAccessIO.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
+            return badRequest("This dataset does not have managed Globus storage");
+        }
+            
+        if (permissionSvc.requestOn(createDataverseRequest(authUser), dataset)
+                .canIssue(UpdateDatasetVersionCommand.class)) {
+
+            JsonObject params = JsonUtil.getJsonObject(jsonBody);
+            String principal = params.getString("principal");
+
+            // Async Call
+            int status = globusService.givePermission("identity", principal, "rw", dataset);
+            switch (status) {
+            case 201:
+                return ok("Permission Granted");
+            case 400:
+                return badRequest("Unable to grant permission");
+            case 409:
+                return conflict("Permission already exists");
+            default:
+                return error(null, "Unexpected error when granting permission");
+            }
+        } else {
+            return forbidden("User doesn't have permission to upload to this dataset");
+        }
 
     }
 

From c724094dcfffaa83c61f415d572e2e5a8958cef0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 3 Oct 2023 17:03:57 -0400
Subject: [PATCH 509/922] fix constructor, reformat

---
 .../iq/dataverse/globus/GlobusEndpoint.java   | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java
index d1e5d19a592..7e555935e2e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusEndpoint.java
@@ -5,27 +5,34 @@ public class GlobusEndpoint {
     private String id;
     private String clientToken;
     private String basePath;
- 
-    
+
     public GlobusEndpoint(String id, String clientToken, String basePath) {
-        
+        this.id = id;
+        this.clientToken = clientToken;
+        this.basePath = basePath;
     }
-        public String getId() {
+
+    public String getId() {
         return id;
     }
-        public void setId(String id) {
+
+    public void setId(String id) {
         this.id = id;
     }
-        public String getClientToken() {
+
+    public String getClientToken() {
         return clientToken;
     }
-        public void setClientToken(String clientToken) {
+
+    public void setClientToken(String clientToken) {
         this.clientToken = clientToken;
     }
-        public String getBasePath() {
+
+    public String getBasePath() {
         return basePath;
     }
-        public void setBasePath(String basePath) {
+
+    public void setBasePath(String basePath) {
         this.basePath = basePath;
     }
 }
\ No newline at end of file

From ed87e0640788278b5af838ba98efd72413d2586d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 3 Oct 2023 17:04:51 -0400
Subject: [PATCH 510/922] start to monitor access rule changes

---
 .../dataverse/globus/GlobusServiceBean.java   | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 910ee796e0e..ad20b90971b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -206,10 +206,21 @@ public int givePermission(String principalType, String principal, String perm, D
             result = makeRequest(url, "Bearer", endpoint.getClientToken(), "POST",
                     gson.toJson(permissions));
 
-            if (result.status == 400) {
+            switch (result.status) {
+            case 400:
+
                 logger.severe("Path " + permissions.getPath() + " is not valid");
-            } else if (result.status == 409) {
+                break;
+            case 409:
                 logger.warning("ACL already exists or Endpoint ACL already has the maximum number of access rules");
+                break;
+            case 201:
+                JsonObject response = JsonUtil.getJsonObject(result.jsonResponse);
+                if (response != null && response.containsKey("access_id")) {
+                    permissions.setId(response.getString("access_id"));
+                    monitorTemporaryPermissions(permissions, endpoint);
+                    logger.info("Access rule " + permissions.getId() + " was created successfully");
+                }
             }
 
             return result.status;
@@ -226,9 +237,13 @@ public int givePermission(String principalType, String principal, String perm, D
                 logger.warning("ACL already exists or Endpoint ACL already has the maximum number of access rules");
             }
             logger.info("Result status " + result.status);
+            return result.status;
         }
+    }
 
-        return result.status;
+    private void monitorTemporaryPermissions(Permissions permissions, GlobusEndpoint endpoint) {
+        // TODO Auto-generated method stub
+        
     }
 
     public boolean getSuccessfulTransfers(AccessToken clientTokenUser, String taskId) throws MalformedURLException {
@@ -324,6 +339,7 @@ public static MakeRequestResponse makeRequest(URL url, String authType, String a
             // Basic
             // NThjMGYxNDQtN2QzMy00ZTYzLTk3MmUtMjljNjY5YzJjNGJiOktzSUVDMDZtTUxlRHNKTDBsTmRibXBIbjZvaWpQNGkwWVVuRmQyVDZRSnc9
             logger.info(authType + " " + authCode);
+            logger.info("For URL: " + url.toString());
             connection.setRequestProperty("Authorization", authType + " " + authCode);
             // connection.setRequestProperty("Content-Type",
             // "application/x-www-form-urlencoded");
@@ -333,6 +349,7 @@ public static MakeRequestResponse makeRequest(URL url, String authType, String a
                 connection.setRequestProperty("Accept", "application/json");
                 logger.info(jsonString);
                 connection.setDoOutput(true);
+
                 OutputStreamWriter wr = new OutputStreamWriter(connection.getOutputStream());
                 wr.write(jsonString);
                 wr.flush();
@@ -1318,7 +1335,7 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         String clientToken = accessToken.getOtherTokens().get(0).getAccessToken();
-
+logger.info("clientToken: " + clientToken);
         endpoint = new GlobusEndpoint(endpointId, clientToken, directoryPath);
 
         return endpoint;

From 6f957a79a5744c83216e40b83370bcffc47be418 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 3 Oct 2023 17:39:38 -0400
Subject: [PATCH 511/922] user is required

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b9a104d8eaa..52ca9cd748f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -580,13 +580,14 @@ public Response getVersionMetadataBlock(@Context ContainerRequestContext crc,
      * @return
      */
     @GET
+    @AuthRequired
     @Path("{id}/versions/{versionId}/linkset")
-    public Response getLinkset(@PathParam("id") String datasetId, @PathParam("versionId") String versionId,
+    public Response getLinkset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId,
             @Context UriInfo uriInfo, @Context HttpHeaders headers) {
         if (":draft".equals(versionId)) {
             return badRequest("Signposting is not supported on the :draft version");
         }
-        DataverseRequest req = createDataverseRequest(null);
+        DataverseRequest req = createDataverseRequest(getRequestUser(crc));
         try {
             DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             return Response

From 7b1f5d03a50249dffe21ef2285543e09cc95c71f Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 09:39:46 +0100
Subject: [PATCH 512/922] Fixed: calling new getRequester method instead of old
 getAuthenticatedUser of FileAccessRequest

---
 src/main/java/edu/harvard/iq/dataverse/api/Access.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 014604bff82..3c226e68472 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1697,7 +1697,7 @@ public Response getUserFileAccessRequested(@Context ContainerRequestContext crc,
         boolean fileAccessRequested = false;
         List<FileAccessRequest> requests = dataFile.getFileAccessRequests();
         for (FileAccessRequest fileAccessRequest : requests) {
-            if (fileAccessRequest.getAuthenticatedUser().getId().equals(requestAuthenticatedUser.getId())) {
+            if (fileAccessRequest.getRequester().getId().equals(requestAuthenticatedUser.getId())) {
                 fileAccessRequested = true;
                 break;
             }

From ed903b28aa77cdc516946f47402288ea3be63f86 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 13:03:52 +0100
Subject: [PATCH 513/922] Added: getVersionFiles filter by tabular tag name
 (pending IT)

---
 .../edu/harvard/iq/dataverse/DataFileTag.java |  2 +-
 .../DatasetVersionFilesServiceBean.java       | 10 +++-
 .../harvard/iq/dataverse/api/Datasets.java    |  3 +-
 .../edu/harvard/iq/dataverse/api/Files.java   | 34 +++++++++++
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 59 +++++++++++++------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 19 +++++-
 6 files changed, 105 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
index f4f66d3c874..351c4032939 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
@@ -58,7 +58,7 @@ public enum TagType {Survey, TimeSeries, Panel, Event, Genomics, Network, Geospa
     
     private static final Map<TagType, String> TagTypeToLabels = new HashMap<>();
     
-    private static final Map<String, TagType> TagLabelToTypes = new HashMap<>();
+    public static final Map<String, TagType> TagLabelToTypes = new HashMap<>();
     
     
     static {
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index a547a216ad5..6006d937100 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.QDataFileCategory;
+import edu.harvard.iq.dataverse.QDataFileTag;
 import edu.harvard.iq.dataverse.QDvObject;
 import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
@@ -26,6 +27,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
+
 @Stateless
 @Named
 public class DatasetVersionFilesServiceBean implements Serializable {
@@ -36,6 +39,7 @@ public class DatasetVersionFilesServiceBean implements Serializable {
     private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
     private final QDvObject dvObject = QDvObject.dvObject;
     private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
+    private final QDataFileTag dataFileTag = QDataFileTag.dataFileTag;
 
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
@@ -126,11 +130,12 @@ public Map<DataFileAccessStatus, Long> getFileMetadataCountPerAccessStatus(Datas
      * @param contentType    for retrieving only files with this content type
      * @param accessStatus   for retrieving only files with this DataFileAccessStatus
      * @param categoryName   for retrieving only files categorized with this category name
+     * @param tabularTagName for retrieving only files categorized with this tabular tag name
      * @param searchText     for retrieving only files that contain the specified text within their labels or descriptions
      * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
      * @return a FileMetadata list from the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
         JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
 
         if (contentType != null) {
@@ -142,6 +147,9 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         if (categoryName != null) {
             baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
         }
+        if (tabularTagName != null) {
+            baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
+        }
         if (searchText != null && !searchText.isEmpty()) {
             searchText = searchText.trim().toLowerCase();
             baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index bd0f29463f5..b3be55399d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -499,6 +499,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @QueryParam("contentType") String contentType,
                                     @QueryParam("accessStatus") String accessStatus,
                                     @QueryParam("categoryName") String categoryName,
+                                    @QueryParam("tabularTagName") String tabularTagName,
                                     @QueryParam("searchText") String searchText,
                                     @QueryParam("orderCriteria") String orderCriteria,
                                     @Context UriInfo uriInfo,
@@ -517,7 +518,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
             }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, searchText, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, tabularTagName, searchText, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index d83a55a2914..9296220766b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -4,6 +4,7 @@
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
+import edu.harvard.iq.dataverse.DataFileTag;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetLock;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
@@ -891,6 +892,39 @@ public Response setFileCategories(@Context ContainerRequestContext crc, @PathPar
         }, getRequestUser(crc));
     }
 
+    @POST
+    @AuthRequired
+    @Path("{id}/metadata/tabularTags")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            if (!dataFile.isTabularData()) {
+                return badRequest("This operation is only available for tabular files.");
+            }
+            jakarta.json.JsonObject jsonObject;
+            try (StringReader stringReader = new StringReader(jsonBody)) {
+                jsonObject = Json.createReader(stringReader).readObject();
+                JsonArray requestedTabularTagsJson = jsonObject.getJsonArray("tabularTags");
+                for (JsonValue jsonValue : requestedTabularTagsJson) {
+                    JsonString jsonString = (JsonString) jsonValue;
+                    DataFileTag tag = new DataFileTag();
+                    try {
+                        tag.setTypeByLabel(jsonString.getString());
+                    } catch (IllegalArgumentException iax) {
+                        return badRequest(iax.getMessage());
+                    }
+                    tag.setDataFile(dataFile);
+                    dataFile.addTag(tag);
+                }
+                execCommand(new UpdateDatasetVersionCommand(dataFile.getOwner(), req));
+                return ok("Tabular tags of file " + dataFileId + " updated.");
+            } catch (JsonParsingException jpe) {
+                return badRequest("Error parsing Json: " + jpe.getMessage());
+            }
+        }, getRequestUser(crc));
+    }
+
     @GET
     @AuthRequired
     @Path("{id}/hasBeenDeleted")
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 6f103df3fe8..b9f09cc7c07 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -75,6 +75,7 @@
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 
+import static java.lang.Thread.sleep;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.hamcrest.CoreMatchers;
@@ -3266,8 +3267,7 @@ public void getDatasetVersionCitation() {
                 .body("data.message", containsString("DRAFT VERSION"));
     }
 
-    @Test
-    public void getVersionFiles() throws IOException {
+    public void getVersionFiles() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3299,7 +3299,7 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3313,7 +3313,7 @@ public void getVersionFiles() throws IOException {
         String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
 
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3324,7 +3324,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(testPageSize, fileMetadatasCount);
 
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3334,7 +3334,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3345,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,13 +3390,13 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
 
         // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, apiToken);
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, null, apiToken);
 
         getVersionFilesResponseContentType.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3412,7 +3412,7 @@ public void getVersionFiles() throws IOException {
         setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory));
         setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, apiToken);
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, null, apiToken);
 
         getVersionFilesResponseCategoryName.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3427,7 +3427,7 @@ public void getVersionFiles() throws IOException {
         restrictFileResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponseRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3452,7 +3452,7 @@ public void getVersionFiles() throws IOException {
         createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3461,7 +3461,7 @@ public void getVersionFiles() throws IOException {
         fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
 
-        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3471,7 +3471,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Access Status Public
-        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, apiToken);
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponsePublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3483,7 +3483,7 @@ public void getVersionFiles() throws IOException {
         assertEquals(3, fileMetadatasCount);
 
         // Test Search Text
-        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, "test_1", null, apiToken);
+        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, "test_1", null, apiToken);
 
         getVersionFilesResponseSearchText.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3491,6 +3491,29 @@ public void getVersionFiles() throws IOException {
 
         fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
+
+        // Test Tabular Tag Name
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Ensure tabular file is ingested
+        sleep(2000);
+
+        String tabularTagName = "Survey";
+        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(tabularTagName));
+        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseTabularTagName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, tabularTagName, null, null, apiToken);
+
+        getVersionFilesResponseTabularTagName.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo("test.tab"));
+
+        fileMetadatasCount = getVersionFilesResponseTabularTagName.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
     }
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index d243d3c47f2..8e333451c8d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3276,7 +3276,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String searchText, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3295,6 +3295,9 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (categoryName != null) {
             requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
         }
+        if (tabularTagName != null) {
+            requestSpecification = requestSpecification.queryParam("tabularTagName", tabularTagName);
+        }
         if (searchText != null) {
             requestSpecification = requestSpecification.queryParam("searchText", searchText);
         }
@@ -3375,6 +3378,20 @@ static Response setFileCategories(String dataFileId, String apiToken, List<Strin
                 .post("/api/files/" + dataFileId + "/metadata/categories");
     }
 
+    static Response setFileTabularTags(String dataFileId, String apiToken, List<String> tabularTags) {
+        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
+        for (String tabularTag : tabularTags) {
+            jsonArrayBuilder.add(tabularTag);
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("tabularTags", jsonArrayBuilder);
+        String jsonString = jsonObjectBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .post("/api/files/" + dataFileId + "/metadata/tabularTags");
+    }
+
     static Response deleteFileInDataset(Integer fileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)

From c00ad7915f42c3893fea0ada09f7da3eddaa7026 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 13:38:27 +0100
Subject: [PATCH 514/922] Added: IT and minor refactor for setFileTabularTags
 endpoint

---
 .../edu/harvard/iq/dataverse/api/Files.java   | 10 ++--
 src/main/java/propertyFiles/Bundle.properties |  1 +
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 51 +++++++++++++++++++
 3 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 9296220766b..8a9abe68d85 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -853,17 +853,17 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, @PathPar
         try {
             dataFile = findDataFileOrDie(dataFileId);
         } catch (WrappedResponse e) {
-            return error(Response.Status.NOT_FOUND, "File not found for given id.");
+            return notFound("File not found for given id.");
         }
         if (dataFile.isRestricted() || FileUtil.isActivelyEmbargoed(dataFile)) {
             DataverseRequest dataverseRequest = createDataverseRequest(getRequestUser(crc));
             boolean hasPermissionToDownloadFile = permissionSvc.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
             if (!hasPermissionToDownloadFile) {
-                return error(FORBIDDEN, "Insufficient permissions to access the requested information.");
+                return forbidden("Insufficient permissions to access the requested information.");
             }
         }
         if (!dataFile.isTabularData()) {
-            return error(BAD_REQUEST, "This operation is only available for tabular files.");
+            return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"));
         }
         return ok(jsonDT(dataFile.getDataTables()));
     }
@@ -887,7 +887,7 @@ public Response setFileCategories(@Context ContainerRequestContext crc, @PathPar
                 execCommand(new UpdateDatasetVersionCommand(fileMetadata.getDataFile().getOwner(), req));
                 return ok("Categories of file " + dataFileId + " updated.");
             } catch (JsonParsingException jpe) {
-                return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage());
+                return badRequest("Error parsing Json: " + jpe.getMessage());
             }
         }, getRequestUser(crc));
     }
@@ -900,7 +900,7 @@ public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathPa
         return response(req -> {
             DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
             if (!dataFile.isTabularData()) {
-                return badRequest("This operation is only available for tabular files.");
+                return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"));
             }
             jakarta.json.JsonObject jsonObject;
             try (StringReader stringReader = new StringReader(jsonBody)) {
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index ac725caf1b2..7b4befcca36 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2620,6 +2620,7 @@ admin.api.deleteUser.success=Authenticated User {0} deleted.
 #Files.java
 files.api.metadata.update.duplicateFile=Filename already exists at {0}
 files.api.no.draft=No draft available for this file
+files.api.only.tabular.supported=This operation is only available for tabular files.
 
 #Datasets.java
 datasets.api.updatePIDMetadata.failure.dataset.must.be.released=Modify Registration Metadata must be run on a published dataset.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index d432b08388d..bfe856a8d18 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2247,6 +2247,57 @@ public void testSetFileCategories() {
                 .statusCode(OK.getStatusCode());
     }
 
+    @Test
+    public void testSetFileTabularTags() throws InterruptedException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload tabular file
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Ensure tabular file is ingested
+        sleep(2000);
+
+        // Set tabular tags
+        String testTabularTag1 = "Survey";
+        String testTabularTag2 = "Genomics";
+        List<String> testTabularTags = List.of(testTabularTag1, testTabularTag2);
+        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, testTabularTags);
+        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get file data and check for new categories
+        Response getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
+        getFileDataResponse.then().assertThat()
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
+                .statusCode(OK.getStatusCode());
+
+        // Set invalid tabular tag
+        String testInvalidTabularTag = "Invalid";
+        setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(testInvalidTabularTag));
+        setFileTabularTagsResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+
+        // Get file data and check categories are unaltered
+        getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
+        getFileDataResponse.then().assertThat()
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
+                .statusCode(OK.getStatusCode());
+    }
+
     @Test
     public void testGetHasBeenDeleted() {
         Response createUser = UtilIT.createRandomUser();

From 224fb49043f1fb59c3dfecea164bd29e9ccf16d4 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 14:51:12 +0100
Subject: [PATCH 515/922] Added: setFileTabularTags API endpoint tweaks, IT and
 docs

---
 doc/sphinx-guides/source/api/native-api.rst   | 55 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/DataFile.java    | 24 ++++++--
 .../edu/harvard/iq/dataverse/api/Files.java   |  7 +--
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 14 ++++-
 4 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index cf869d338ca..d4e2e4cd178 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3060,6 +3060,61 @@ The fully expanded example above (without environment variables) looks like this
 
 Note that if the specified categories do not exist, they will be created.
 
+Updating File Tabular Tags
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates the tabular tags for an existing tabular file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the tabular tag names.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
+    "$SERVER_URL/api/files/$ID/metadata/tabularTags"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
+    "http://demo.dataverse.org/api/files/24/metadata/tabularTags"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
+    "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000"
+
+Note that the specified tabular tags must be valid. The supported tags are:
+
+* ``Survey``
+* ``Time Series``
+* ``Panel``
+* ``Event``
+* ``Genomics``
+* ``Network``
+* ``Geospatial``
+
 Editing Variable Level Metadata
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index ba13729793e..407282a5372 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -386,7 +386,17 @@ public JsonArrayBuilder getTagLabelsAsJsonArrayBuilder(){
     public void setTags(List<DataFileTag> dataFileTags) {
         this.dataFileTags = dataFileTags;
     }
-    
+
+    public void addUniqueTagByLabel(String tagLabel) throws IllegalArgumentException {
+        if (tagExists(tagLabel)) {
+            return;
+        }
+        DataFileTag tag = new DataFileTag();
+        tag.setTypeByLabel(tagLabel);
+        tag.setDataFile(this);
+        addTag(tag);
+    }
+
     public void addTag(DataFileTag tag) {
         if (dataFileTags == null) {
             dataFileTags = new ArrayList<>();
@@ -1093,8 +1103,12 @@ public String getTargetUrl() {
         return DataFile.TARGET_URL;
     }
 
+    private boolean tagExists(String tagLabel) {
+        for (DataFileTag dataFileTag : dataFileTags) {
+            if (dataFileTag.getTypeLabel().equals(tagLabel)) {
+                return true;
+            }
+        }
+        return false;
+    }
 } // end of class
-    
-
-    
-
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 8a9abe68d85..ad24d81d996 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -908,14 +908,11 @@ public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathPa
                 JsonArray requestedTabularTagsJson = jsonObject.getJsonArray("tabularTags");
                 for (JsonValue jsonValue : requestedTabularTagsJson) {
                     JsonString jsonString = (JsonString) jsonValue;
-                    DataFileTag tag = new DataFileTag();
                     try {
-                        tag.setTypeByLabel(jsonString.getString());
-                    } catch (IllegalArgumentException iax) {
+                        dataFile.addUniqueTagByLabel(jsonString.getString());
+                    } catch (IllegalArgumentException iax){
                         return badRequest(iax.getMessage());
                     }
-                    tag.setDataFile(dataFile);
-                    dataFile.addTag(tag);
                 }
                 execCommand(new UpdateDatasetVersionCommand(dataFile.getOwner(), req));
                 return ok("Tabular tags of file " + dataFileId + " updated.");
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index bfe856a8d18..af678905fe1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2274,28 +2274,36 @@ public void testSetFileTabularTags() throws InterruptedException {
         // Set tabular tags
         String testTabularTag1 = "Survey";
         String testTabularTag2 = "Genomics";
-        List<String> testTabularTags = List.of(testTabularTag1, testTabularTag2);
+        // We repeat one to test that it is not duplicated
+        String testTabularTag3 = "Genomics";
+        List<String> testTabularTags = List.of(testTabularTag1, testTabularTag2, testTabularTag3);
         Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, testTabularTags);
         setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Get file data and check for new categories
+        // Get file data and check for new tabular tags
         Response getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
         getFileDataResponse.then().assertThat()
                 .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
                 .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
                 .statusCode(OK.getStatusCode());
 
+        int actualTabularTagsCount = getFileDataResponse.jsonPath().getList("data.dataFile.tabularTags").size();
+        assertEquals(2, actualTabularTagsCount);
+
         // Set invalid tabular tag
         String testInvalidTabularTag = "Invalid";
         setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(testInvalidTabularTag));
         setFileTabularTagsResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
 
-        // Get file data and check categories are unaltered
+        // Get file data and check tabular tags are unaltered
         getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
         getFileDataResponse.then().assertThat()
                 .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
                 .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
                 .statusCode(OK.getStatusCode());
+
+        actualTabularTagsCount = getFileDataResponse.jsonPath().getList("data.dataFile.tabularTags").size();
+        assertEquals(2, actualTabularTagsCount);
     }
 
     @Test

From 122565a4f1afac9f5fe0d85a3c6f08517eb530c4 Mon Sep 17 00:00:00 2001
From: Kevin Condon <kcondon@hmdc.harvard.edu>
Date: Wed, 4 Oct 2023 09:57:22 -0400
Subject: [PATCH 516/922] Revert "Dataset files API extension for filters"

---
 .../9714-files-api-extension-filters.md       |  14 -
 .../9785-files-api-extension-search-text.md   |   3 -
 .../9834-files-api-extension-counts.md        |   6 -
 ...oad-extension-new-file-access-endpoints.md |  14 -
 doc/sphinx-guides/source/api/dataaccess.rst   |  12 -
 doc/sphinx-guides/source/api/native-api.rst   | 165 +-----------
 modules/dataverse-parent/pom.xml              |   3 -
 pom.xml                                       |  14 -
 .../edu/harvard/iq/dataverse/DataFileTag.java |   2 +-
 .../DatasetVersionFilesServiceBean.java       | 241 ------------------
 .../dataverse/DatasetVersionServiceBean.java  |  74 ++++++
 .../edu/harvard/iq/dataverse/api/Access.java  |  59 ++---
 .../harvard/iq/dataverse/api/Datasets.java    |  50 +---
 .../edu/harvard/iq/dataverse/api/Files.java   |  90 +------
 .../iq/dataverse/util/json/JsonPrinter.java   |  21 --
 src/main/java/propertyFiles/Bundle.properties |   1 -
 .../harvard/iq/dataverse/api/AccessIT.java    |  47 ++--
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 240 +++--------------
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 145 +----------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  83 +-----
 20 files changed, 174 insertions(+), 1110 deletions(-)
 delete mode 100644 doc/release-notes/9714-files-api-extension-filters.md
 delete mode 100644 doc/release-notes/9785-files-api-extension-search-text.md
 delete mode 100644 doc/release-notes/9834-files-api-extension-counts.md
 delete mode 100644 doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
 delete mode 100644 src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java

diff --git a/doc/release-notes/9714-files-api-extension-filters.md b/doc/release-notes/9714-files-api-extension-filters.md
deleted file mode 100644
index 034230efe61..00000000000
--- a/doc/release-notes/9714-files-api-extension-filters.md
+++ /dev/null
@@ -1,14 +0,0 @@
-The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by:
-
-- Access status: through the `accessStatus` query parameter, which supports the following values:
-
-  - Public
-  - Restricted
-  - EmbargoedThenRestricted
-  - EmbargoedThenPublic
-
-
-- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
-
-
-- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
diff --git a/doc/release-notes/9785-files-api-extension-search-text.md b/doc/release-notes/9785-files-api-extension-search-text.md
deleted file mode 100644
index fb185e1c7af..00000000000
--- a/doc/release-notes/9785-files-api-extension-search-text.md
+++ /dev/null
@@ -1,3 +0,0 @@
-The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by search text through the `searchText` query parameter.
-
-The search will be applied to the labels and descriptions of the dataset files.
diff --git a/doc/release-notes/9834-files-api-extension-counts.md b/doc/release-notes/9834-files-api-extension-counts.md
deleted file mode 100644
index 3ec15d8bd36..00000000000
--- a/doc/release-notes/9834-files-api-extension-counts.md
+++ /dev/null
@@ -1,6 +0,0 @@
-Implemented the following new endpoints:
-
-- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name). 
-
-
-- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
diff --git a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
deleted file mode 100644
index f306ae2ab80..00000000000
--- a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
+++ /dev/null
@@ -1,14 +0,0 @@
-Implemented the following new endpoints:
-
-- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file. 
-
-
-- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
-
-
-In addition, the DataFile API payload has been extended to include the following fields:
-
-- tabularData: Boolean field to know if the DataFile is of tabular type
-
-
-- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner) 
diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 6edd413b7a5..21be702d62b 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -404,18 +404,6 @@ A curl example using an ``id``::
 
     curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests
 
-User Has Requested Access to a File:
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-``/api/access/datafile/{id}/userFileAccessRequested``
-
-This method returns true or false depending on whether or not the calling user has requested access to a particular file.
-
-A curl example using an ``id``::
-
-    curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userFileAccessRequested"
-
-
 Get User Permissions on a File:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index cf869d338ca..5c72937eecb 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -970,45 +970,6 @@ This endpoint supports optional pagination, through the ``limit`` and ``offset``
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
 
-Category name filtering is also optionally supported. To return files to which the requested category has been added.
-
-Usage example:
-
-.. code-block:: bash
-
-  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?categoryName=Data"
-
-Content type filtering is also optionally supported. To return files matching the requested content type.
-
-Usage example:
-
-.. code-block:: bash
-
-  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?contentType=image/png"
-
-Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the files that contain the text searched in one of such fields.
-
-Usage example:
-
-.. code-block:: bash
-
-  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?searchText=word"
-
-File access filtering is also optionally supported. In particular, by the following possible values:
-
-* ``Public``
-* ``Restricted``
-* ``EmbargoedThenRestricted``
-* ``EmbargoedThenPublic``
-
-If no filter is specified, the files will match all of the above categories.
-
-Usage example:
-
-.. code-block:: bash
-
-  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?accessStatus=Public"
-
 Ordering criteria for sorting the results is also optionally supported. In particular, by the following possible values:
 
 * ``NameAZ`` (Default)
@@ -1018,42 +979,14 @@ Ordering criteria for sorting the results is also optionally supported. In parti
 * ``Size``
 * ``Type``
 
+Please note that these values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
 Usage example:
 
 .. code-block:: bash
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?orderCriteria=Newest"
 
-Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
-
-Keep in mind that you can combine all of the above query params depending on the results you are looking for.
-
-Get File Counts in a Dataset
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Get file counts, for the given dataset and version.
-
-The returned file counts are based on different criteria:
-
-- Total (The total file count)
-- Per content type
-- Per category name
-- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic)
-
-.. code-block:: bash
-
-  export SERVER_URL=https://demo.dataverse.org
-  export ID=24
-  export VERSION=1.0
-
-  curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/files/counts"
-
-The fully expanded example above (without environment variables) looks like this:
-
-.. code-block:: bash
-
-  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts"
-
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -2899,13 +2832,13 @@ A curl example using an ``ID``
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/downloadCount"
+  curl "$SERVER_URL/api/files/$ID/downloadCount"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/downloadCount"
+  curl "https://demo.dataverse.org/api/files/24/downloadCount"
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -2915,53 +2848,16 @@ A curl example using a ``PERSISTENT_ID``
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
+  curl "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
+  curl "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
 
 If you are interested in download counts for multiple files, see :doc:`/api/metrics`.
 
-File Has Been Deleted
-~~~~~~~~~~~~~~~~~~~~~
-
-Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
-
-A curl example using an ``ID``
-
-.. code-block:: bash
-
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export ID=24
-
-  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/hasBeenDeleted"
-
-The fully expanded example above (without environment variables) looks like this:
-
-.. code-block:: bash
-
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/hasBeenDeleted"
-
-A curl example using a ``PERSISTENT_ID``
-
-.. code-block:: bash
-
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
-
-  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID"
-
-The fully expanded example above (without environment variables) looks like this:
-
-.. code-block:: bash
-
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET  "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000"
-
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -3011,55 +2907,6 @@ Also note that dataFileTags are not versioned and changes to these will update t
 
 .. _EditingVariableMetadata:
 
-Updating File Metadata Categories
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Updates the categories for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the category names.
-
-Although updating categories can also be done with the previous endpoint, this has been created to be more practical when it is only necessary to update categories and not other metadata fields.
-
-A curl example using an ``ID``
-
-.. code-block:: bash
-
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export ID=24
-
-  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "$SERVER_URL/api/files/$ID/metadata/categories"
-
-The fully expanded example above (without environment variables) looks like this:
-
-.. code-block:: bash
-
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "http://demo.dataverse.org/api/files/24/metadata/categories"
-
-A curl example using a ``PERSISTENT_ID``
-
-.. code-block:: bash
-
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
-
-  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID"
-
-The fully expanded example above (without environment variables) looks like this:
-
-.. code-block:: bash
-
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000"
-
-Note that if the specified categories do not exist, they will be created.
-
 Editing Variable Level Metadata
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 8e0ff2887df..bfa11af6c70 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -200,9 +200,6 @@
         
         <!-- Container related -->
         <fabric8-dmp.version>0.43.4</fabric8-dmp.version>
-
-        <!-- Persistence -->
-        <querydsl.version>5.0.0</querydsl.version>
     </properties>
     
     <pluginRepositories>
diff --git a/pom.xml b/pom.xml
index 909e9ee9b80..e70b723cad5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -252,20 +252,6 @@
             <artifactId>expressly</artifactId>
             <scope>provided</scope>
         </dependency>
-
-        <dependency>
-            <groupId>com.querydsl</groupId>
-            <artifactId>querydsl-apt</artifactId>
-            <version>${querydsl.version}</version>
-            <classifier>jakarta</classifier>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.querydsl</groupId>
-            <artifactId>querydsl-jpa</artifactId>
-            <version>${querydsl.version}</version>
-            <classifier>jakarta</classifier>
-        </dependency>
         
         <dependency>
             <groupId>commons-io</groupId>
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
index 351c4032939..f4f66d3c874 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
@@ -58,7 +58,7 @@ public enum TagType {Survey, TimeSeries, Panel, Event, Genomics, Network, Geospa
     
     private static final Map<TagType, String> TagTypeToLabels = new HashMap<>();
     
-    public static final Map<String, TagType> TagLabelToTypes = new HashMap<>();
+    private static final Map<String, TagType> TagLabelToTypes = new HashMap<>();
     
     
     static {
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
deleted file mode 100644
index 6006d937100..00000000000
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ /dev/null
@@ -1,241 +0,0 @@
-package edu.harvard.iq.dataverse;
-
-import edu.harvard.iq.dataverse.QDataFileCategory;
-import edu.harvard.iq.dataverse.QDataFileTag;
-import edu.harvard.iq.dataverse.QDvObject;
-import edu.harvard.iq.dataverse.QEmbargo;
-import edu.harvard.iq.dataverse.QFileMetadata;
-
-import com.querydsl.core.Tuple;
-import com.querydsl.core.types.dsl.BooleanExpression;
-import com.querydsl.core.types.dsl.CaseBuilder;
-import com.querydsl.core.types.dsl.DateExpression;
-import com.querydsl.core.types.dsl.DateTimeExpression;
-
-import com.querydsl.jpa.impl.JPAQuery;
-import com.querydsl.jpa.impl.JPAQueryFactory;
-
-import jakarta.ejb.Stateless;
-import jakarta.inject.Named;
-import jakarta.persistence.EntityManager;
-import jakarta.persistence.PersistenceContext;
-
-import java.io.Serializable;
-import java.sql.Timestamp;
-import java.time.LocalDate;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
-
-@Stateless
-@Named
-public class DatasetVersionFilesServiceBean implements Serializable {
-
-    @PersistenceContext(unitName = "VDCNet-ejbPU")
-    private EntityManager em;
-
-    private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
-    private final QDvObject dvObject = QDvObject.dvObject;
-    private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
-    private final QDataFileTag dataFileTag = QDataFileTag.dataFileTag;
-
-    /**
-     * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
-     */
-    public enum FileMetadatasOrderCriteria {
-        NameAZ, NameZA, Newest, Oldest, Size, Type
-    }
-
-    /**
-     * Status of the particular DataFile based on active embargoes and restriction state used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
-     */
-    public enum DataFileAccessStatus {
-        Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
-    }
-
-    /**
-     * Given a DatasetVersion, returns its total file metadata count
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @return long value of total file metadata count
-     */
-    public long getFileMetadataCount(DatasetVersion datasetVersion) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).stream().count();
-    }
-
-    /**
-     * Given a DatasetVersion, returns its file metadata count per content type
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @return Map<String, Long> of file metadata counts per content type
-     */
-    public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        List<Tuple> contentTypeOccurrences = queryFactory
-                .select(fileMetadata.dataFile.contentType, fileMetadata.count())
-                .from(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                .groupBy(fileMetadata.dataFile.contentType).fetch();
-        Map<String, Long> result = new HashMap<>();
-        for (Tuple occurrence : contentTypeOccurrences) {
-            result.put(occurrence.get(fileMetadata.dataFile.contentType), occurrence.get(fileMetadata.count()));
-        }
-        return result;
-    }
-
-    /**
-     * Given a DatasetVersion, returns its file metadata count per category name
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @return Map<String, Long> of file metadata counts per category name
-     */
-    public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        List<Tuple> categoryNameOccurrences = queryFactory
-                .select(dataFileCategory.name, fileMetadata.count())
-                .from(dataFileCategory, fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.fileCategories.contains(dataFileCategory)))
-                .groupBy(dataFileCategory.name).fetch();
-        Map<String, Long> result = new HashMap<>();
-        for (Tuple occurrence : categoryNameOccurrences) {
-            result.put(occurrence.get(dataFileCategory.name), occurrence.get(fileMetadata.count()));
-        }
-        return result;
-    }
-
-    /**
-     * Given a DatasetVersion, returns its file metadata count per DataFileAccessStatus
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @return Map<DataFileAccessStatus, Long> of file metadata counts per DataFileAccessStatus
-     */
-    public Map<DataFileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) {
-        Map<DataFileAccessStatus, Long> allCounts = new HashMap<>();
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Public);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Restricted);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenPublic);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenRestricted);
-        return allCounts;
-    }
-
-    /**
-     * Returns a FileMetadata list of files in the specified DatasetVersion
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @param limit          for pagination, can be null
-     * @param offset         for pagination, can be null
-     * @param contentType    for retrieving only files with this content type
-     * @param accessStatus   for retrieving only files with this DataFileAccessStatus
-     * @param categoryName   for retrieving only files categorized with this category name
-     * @param tabularTagName for retrieving only files categorized with this tabular tag name
-     * @param searchText     for retrieving only files that contain the specified text within their labels or descriptions
-     * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
-     * @return a FileMetadata list from the specified DatasetVersion
-     */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
-
-        if (contentType != null) {
-            baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
-        }
-        if (accessStatus != null) {
-            baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
-        }
-        if (categoryName != null) {
-            baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
-        }
-        if (tabularTagName != null) {
-            baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
-        }
-        if (searchText != null && !searchText.isEmpty()) {
-            searchText = searchText.trim().toLowerCase();
-            baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
-        }
-
-        applyOrderCriteriaToGetFileMetadatasQuery(baseQuery, orderCriteria);
-
-        if (limit != null) {
-            baseQuery.limit(limit);
-        }
-        if (offset != null) {
-            baseQuery.offset(offset);
-        }
-
-        return baseQuery.fetch();
-    }
-
-    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<DataFileAccessStatus, Long> totalCounts, DataFileAccessStatus dataFileAccessStatus) {
-        long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus);
-        if (fileMetadataCount > 0) {
-            totalCounts.put(dataFileAccessStatus, fileMetadataCount);
-        }
-    }
-
-    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, DataFileAccessStatus accessStatus) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        return queryFactory
-                .selectFrom(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus)))
-                .stream().count();
-    }
-
-    private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
-        if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) {
-            baseQuery.from(dvObject).where(dvObject.id.eq(fileMetadata.dataFile.id));
-        }
-        return baseQuery;
-    }
-
-    private BooleanExpression createGetFileMetadatasAccessStatusExpression(DataFileAccessStatus accessStatus) {
-        QEmbargo embargo = fileMetadata.dataFile.embargo;
-        BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
-        BooleanExpression inactivelyEmbargoedExpression = embargo.isNull();
-        BooleanExpression accessStatusExpression;
-        switch (accessStatus) {
-            case EmbargoedThenRestricted:
-                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
-                break;
-            case EmbargoedThenPublic:
-                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
-                break;
-            case Restricted:
-                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
-                break;
-            case Public:
-                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
-                break;
-            default:
-                throw new IllegalStateException("Unexpected value: " + accessStatus);
-        }
-        return accessStatusExpression;
-    }
-
-    private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
-        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
-        switch (orderCriteria) {
-            case NameZA:
-                query.orderBy(fileMetadata.label.desc());
-                break;
-            case Newest:
-                query.orderBy(orderByLifetimeExpression.desc());
-                break;
-            case Oldest:
-                query.orderBy(orderByLifetimeExpression.asc());
-                break;
-            case Size:
-                query.orderBy(fileMetadata.dataFile.filesize.asc());
-                break;
-            case Type:
-                query.orderBy(fileMetadata.dataFile.contentType.asc());
-                break;
-            default:
-                query.orderBy(fileMetadata.label.asc());
-                break;
-        }
-    }
-}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 5c43001dcb5..6f087f9eabc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -49,6 +49,22 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
+            + " WHERE fm.datasetVersion.id=:datasetVersionId"
+            + " ORDER BY fm.label";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = dvo.id"
+            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.filesize";
+    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
+            + " WHERE fm.datasetVersion.id = :datasetVersionId"
+            + " AND fm.dataFile.id = df.id"
+            + " ORDER BY df.contentType";
+
     @EJB
     DatasetServiceBean datasetService;
     
@@ -150,6 +166,18 @@ public DatasetVersion getDatasetVersion(){
         }                
     } // end RetrieveDatasetVersionResponse
 
+    /**
+     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
+     */
+    public enum FileMetadatasOrderCriteria {
+        NameAZ,
+        NameZA,
+        Newest,
+        Oldest,
+        Size,
+        Type
+    }
+
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -1224,4 +1252,50 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
+
+    /**
+     * Returns a FileMetadata list of files in the specified DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param limit for pagination, can be null
+     * @param offset for pagination, can be null
+     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
+     * @return a FileMetadata list of the specified DatasetVersion
+     */
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
+        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
+                .setParameter("datasetVersionId", datasetVersion.getId());
+        if (limit != null) {
+            query.setMaxResults(limit);
+        }
+        if (offset != null) {
+            query.setFirstResult(offset);
+        }
+        return query.getResultList();
+    }
+
+    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
+        String queryString;
+        switch (orderCriteria) {
+            case NameZA:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
+                break;
+            case Newest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
+                break;
+            case Oldest:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
+                break;
+            case Size:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
+                break;
+            case Type:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
+                break;
+            default:
+                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
+                break;
+        }
+        return queryString;
+    }
 } // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 3c226e68472..1aa3f4ffde6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1681,47 +1681,7 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.rejectFailure.noRequest", args));
         }
     }
-
-    @GET
-    @AuthRequired
-    @Path("/datafile/{id}/userFileAccessRequested")
-    public Response getUserFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        AuthenticatedUser requestAuthenticatedUser;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-            requestAuthenticatedUser = getRequestAuthenticatedUserOrDie(crc);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        boolean fileAccessRequested = false;
-        List<FileAccessRequest> requests = dataFile.getFileAccessRequests();
-        for (FileAccessRequest fileAccessRequest : requests) {
-            if (fileAccessRequest.getRequester().getId().equals(requestAuthenticatedUser.getId())) {
-                fileAccessRequested = true;
-                break;
-            }
-        }
-        return ok(fileAccessRequested);
-    }
-
-    @GET
-    @AuthRequired
-    @Path("/datafile/{id}/userPermissions")
-    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        User requestUser = getRequestUser(crc);
-        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
-        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
-        return ok(jsonObjectBuilder);
-    }
-
+    
     // checkAuthorization is a convenience method; it calls the boolean method
     // isAccessAuthorized(), the actual workhorse, tand throws a 403 exception if not.
     
@@ -1989,4 +1949,21 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin
         }
         return redirectUri;
     }
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userPermissions")
+    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        User requestUser = getRequestUser(crc);
+        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
+        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
+        return ok(jsonObjectBuilder);
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b3be55399d8..704ec8f1989 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -236,9 +236,6 @@ public class Datasets extends AbstractApiBean {
     @Inject
     PrivateUrlServiceBean privateUrlService;
 
-    @Inject
-    DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
-
     /**
      * Used to consolidate the way we parse and handle dataset versions.
      * @param <T> 
@@ -487,56 +484,23 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
                     : ok(json(dsv));
         }, getRequestUser(crc));
     }
-
+    
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
-    public Response getVersionFiles(@Context ContainerRequestContext crc,
-                                    @PathParam("id") String datasetId,
-                                    @PathParam("versionId") String versionId,
-                                    @QueryParam("limit") Integer limit,
-                                    @QueryParam("offset") Integer offset,
-                                    @QueryParam("contentType") String contentType,
-                                    @QueryParam("accessStatus") String accessStatus,
-                                    @QueryParam("categoryName") String categoryName,
-                                    @QueryParam("tabularTagName") String tabularTagName,
-                                    @QueryParam("searchText") String searchText,
-                                    @QueryParam("orderCriteria") String orderCriteria,
-                                    @Context UriInfo uriInfo,
-                                    @Context HttpHeaders headers) {
-        return response(req -> {
+    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response( req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
             try {
-                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameAZ;
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            DatasetVersionFilesServiceBean.DataFileAccessStatus dataFileAccessStatus;
-            try {
-                dataFileAccessStatus = accessStatus != null ? DatasetVersionFilesServiceBean.DataFileAccessStatus.valueOf(accessStatus) : null;
-            } catch (IllegalArgumentException e) {
-                return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
-            }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, tabularTagName, searchText, fileMetadatasOrderCriteria)));
+            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
-
-    @GET
-    @AuthRequired
-    @Path("{id}/versions/{versionId}/files/counts")
-    public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return response(req -> {
-            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-            jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
-            jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion)));
-            jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion)));
-            jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion)));
-            return ok(jsonObjectBuilder);
-        }, getRequestUser(crc));
-    }
-
+    
     @GET
     @AuthRequired
     @Path("{id}/dirindex")
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 8a9abe68d85..82811162d52 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -4,7 +4,6 @@
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
-import edu.harvard.iq.dataverse.DataFileTag;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetLock;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
@@ -55,7 +54,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -65,12 +63,15 @@
 import jakarta.ejb.EJBException;
 import jakarta.inject.Inject;
 import jakarta.json.Json;
-import jakarta.json.JsonArray;
-import jakarta.json.JsonString;
-import jakarta.json.JsonValue;
-import jakarta.json.stream.JsonParsingException;
 import jakarta.servlet.http.HttpServletResponse;
-import jakarta.ws.rs.*;
+import jakarta.ws.rs.Consumes;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.PUT;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.HttpHeaders;
@@ -111,8 +112,6 @@ public class Files extends AbstractApiBean {
     MakeDataCountLoggingServiceBean mdcLogService;
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
-    @Inject
-    DataFileServiceBean dataFileServiceBean;
 
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
@@ -853,85 +852,18 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, @PathPar
         try {
             dataFile = findDataFileOrDie(dataFileId);
         } catch (WrappedResponse e) {
-            return notFound("File not found for given id.");
+            return error(Response.Status.NOT_FOUND, "File not found for given id.");
         }
         if (dataFile.isRestricted() || FileUtil.isActivelyEmbargoed(dataFile)) {
             DataverseRequest dataverseRequest = createDataverseRequest(getRequestUser(crc));
             boolean hasPermissionToDownloadFile = permissionSvc.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
             if (!hasPermissionToDownloadFile) {
-                return forbidden("Insufficient permissions to access the requested information.");
+                return error(FORBIDDEN, "Insufficient permissions to access the requested information.");
             }
         }
         if (!dataFile.isTabularData()) {
-            return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"));
+            return error(BAD_REQUEST, "This operation is only available for tabular files.");
         }
         return ok(jsonDT(dataFile.getDataTables()));
     }
-
-    @POST
-    @AuthRequired
-    @Path("{id}/metadata/categories")
-    @Produces(MediaType.APPLICATION_JSON)
-    public Response setFileCategories(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
-        return response(req -> {
-            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
-            jakarta.json.JsonObject jsonObject;
-            try (StringReader stringReader = new StringReader(jsonBody)) {
-                jsonObject = Json.createReader(stringReader).readObject();
-                JsonArray requestedCategoriesJson = jsonObject.getJsonArray("categories");
-                FileMetadata fileMetadata = dataFile.getFileMetadata();
-                for (JsonValue jsonValue : requestedCategoriesJson) {
-                    JsonString jsonString = (JsonString) jsonValue;
-                    fileMetadata.addCategoryByName(jsonString.getString());
-                }
-                execCommand(new UpdateDatasetVersionCommand(fileMetadata.getDataFile().getOwner(), req));
-                return ok("Categories of file " + dataFileId + " updated.");
-            } catch (JsonParsingException jpe) {
-                return badRequest("Error parsing Json: " + jpe.getMessage());
-            }
-        }, getRequestUser(crc));
-    }
-
-    @POST
-    @AuthRequired
-    @Path("{id}/metadata/tabularTags")
-    @Produces(MediaType.APPLICATION_JSON)
-    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
-        return response(req -> {
-            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
-            if (!dataFile.isTabularData()) {
-                return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"));
-            }
-            jakarta.json.JsonObject jsonObject;
-            try (StringReader stringReader = new StringReader(jsonBody)) {
-                jsonObject = Json.createReader(stringReader).readObject();
-                JsonArray requestedTabularTagsJson = jsonObject.getJsonArray("tabularTags");
-                for (JsonValue jsonValue : requestedTabularTagsJson) {
-                    JsonString jsonString = (JsonString) jsonValue;
-                    DataFileTag tag = new DataFileTag();
-                    try {
-                        tag.setTypeByLabel(jsonString.getString());
-                    } catch (IllegalArgumentException iax) {
-                        return badRequest(iax.getMessage());
-                    }
-                    tag.setDataFile(dataFile);
-                    dataFile.addTag(tag);
-                }
-                execCommand(new UpdateDatasetVersionCommand(dataFile.getOwner(), req));
-                return ok("Tabular tags of file " + dataFileId + " updated.");
-            } catch (JsonParsingException jpe) {
-                return badRequest("Error parsing Json: " + jpe.getMessage());
-            }
-        }, getRequestUser(crc));
-    }
-
-    @GET
-    @AuthRequired
-    @Path("{id}/hasBeenDeleted")
-    public Response getHasBeenDeleted(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        return response(req -> {
-            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
-            return ok(dataFileServiceBean.hasBeenDeleted(dataFile));
-        }, getRequestUser(crc));
-    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 1fed0b233e4..a7aa36f179e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -672,14 +672,9 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 //---------------------------------------------
                 .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue()))
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
-                .add("tabularData", df.isTabularData())
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
                 .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
-        Dataset dfOwner = df.getOwner();
-        if (dfOwner != null) {
-            builder.add("fileAccessRequest", dfOwner.isFileAccessRequest());
-        }
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the
@@ -1100,22 +1095,6 @@ public Set<Collector.Characteristics> characteristics() {
         };
     }
 
-    public static JsonObjectBuilder json(Map<String, Long> map) {
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        for (Map.Entry<String, Long> mapEntry : map.entrySet()) {
-            jsonObjectBuilder.add(mapEntry.getKey(), mapEntry.getValue());
-        }
-        return jsonObjectBuilder;
-    }
-
-    public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> map) {
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        for (Map.Entry<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> mapEntry : map.entrySet()) {
-            jsonObjectBuilder.add(mapEntry.getKey().toString(), mapEntry.getValue());
-        }
-        return jsonObjectBuilder;
-    }
-
     public static Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder> toJsonArray() {
         return new Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder>() {
 
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 7b4befcca36..ac725caf1b2 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2620,7 +2620,6 @@ admin.api.deleteUser.success=Authenticated User {0} deleted.
 #Files.java
 files.api.metadata.update.duplicateFile=Filename already exists at {0}
 files.api.no.draft=No draft available for this file
-files.api.only.tabular.supported=This operation is only available for tabular files.
 
 #Datasets.java
 datasets.api.updatePIDMetadata.failure.dataset.must.be.released=Modify Registration Metadata must be run on a published dataset.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index 416caa68566..b1beddd893f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -26,9 +26,11 @@
 
 import static jakarta.ws.rs.core.Response.Status.*;
 import static org.hamcrest.MatcherAssert.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
-import static org.junit.jupiter.api.Assertions.*;
 
 /**
  *
@@ -630,37 +632,28 @@ public void testZipUploadAndDownload() throws IOException {
     }
 
     @Test
-    public void testGetUserFileAccessRequested() {
-        // Create new user
-        Response createUserResponse = UtilIT.createRandomUser();
-        createUserResponse.then().assertThat().statusCode(OK.getStatusCode());
-        String newUserApiToken = UtilIT.getApiTokenFromResponse(createUserResponse);
-
-        String dataFileId = Integer.toString(tabFile3IdRestricted);
-
-        // Call with new user and unrequested access file
-        Response getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken);
-        getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode());
+    public void testGetUserPermissionsOnFile() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
 
-        boolean userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data");
-        assertFalse(userFileAccessRequested);
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
-        // Request file access for the new user
-        Response requestFileAccessResponse = UtilIT.requestFileAccess(dataFileId, newUserApiToken);
-        requestFileAccessResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
-        // Call with new user and requested access file
-        getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken);
-        getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data");
-        assertTrue(userFileAccessRequested);
-    }
+        // Assert user permissions on file
+        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
+        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(testFileId), apiToken);
 
-    @Test
-    public void testGetUserPermissionsOnFile() {
-        // Call with valid file id
-        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(basicFileId), apiToken);
         getUserPermissionsOnFileResponse.then().assertThat().statusCode(OK.getStatusCode());
         boolean canDownloadFile = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canDownloadFile");
         assertTrue(canDownloadFile);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index b9f09cc7c07..3b6d4d1ecdf 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -1,6 +1,6 @@
 package edu.harvard.iq.dataverse.api;
 
-import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
+import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
 import io.restassured.RestAssured;
 
 import static io.restassured.RestAssured.given;
@@ -9,9 +9,6 @@
 import io.restassured.http.ContentType;
 import io.restassured.response.Response;
 
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-import java.util.*;
 import java.util.logging.Logger;
 
 import org.junit.jupiter.api.AfterAll;
@@ -20,6 +17,8 @@
 import org.skyscreamer.jsonassert.JSONAssert;
 import org.junit.jupiter.api.Disabled;
 
+import java.util.List;
+import java.util.Map;
 import jakarta.json.JsonObject;
 
 import static jakarta.ws.rs.core.Response.Status.CREATED;
@@ -40,6 +39,8 @@
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
+import java.util.UUID;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 
@@ -66,7 +67,8 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.Files;
-
+import java.util.ArrayList;
+import java.util.HashMap;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObjectBuilder;
@@ -75,7 +77,6 @@
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 
-import static java.lang.Thread.sleep;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.hamcrest.CoreMatchers;
@@ -116,9 +117,7 @@ public static void setUpClass() {
         Response removeAnonymizedFieldTypeNames = UtilIT.deleteSetting(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
         removeAnonymizedFieldTypeNames.then().assertThat()
                 .statusCode(200);
-
-        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
-
+        
         /* With Dual mode, we can no longer mess with upload methods since native is now required for anything to work
                
         Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl);
@@ -145,9 +144,7 @@ public static void afterClass() {
         Response removeAnonymizedFieldTypeNames = UtilIT.deleteSetting(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
         removeAnonymizedFieldTypeNames.then().assertThat()
                 .statusCode(200);
-
-        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
-
+        
         /* See above
         Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl);
         removeDcmUrl.then().assertThat()
@@ -3267,7 +3264,8 @@ public void getDatasetVersionCitation() {
                 .body("data.message", containsString("DRAFT VERSION"));
     }
 
-    public void getVersionFiles() throws IOException, InterruptedException {
+    @Test
+    public void getVersionFiles() throws IOException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3299,42 +3297,39 @@ public void getVersionFiles() throws IOException, InterruptedException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, null, apiToken);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, apiToken);
+
+        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(testPageSize, fileMetadatasCount);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName1))
                 .body("data[1].label", equalTo(testFileName2));
 
-        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(testPageSize, fileMetadatasCount);
-
-        String testFileId1 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
-        String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
-
         // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, apiToken);
+
+        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(testPageSize, fileMetadatasCount);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName3))
                 .body("data[1].label", equalTo(testFileName4));
 
-        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(testPageSize, fileMetadatasCount);
-
         // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, null, apiToken);
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, apiToken);
+
+        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName5));
 
-        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3340,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3351,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3362,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3373,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3390,186 +3385,9 @@ public void getVersionFiles() throws IOException, InterruptedException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
-
-        // Test Content Type
-        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, null, apiToken);
-
-        getVersionFilesResponseContentType.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName5));
-
-        fileMetadatasCount = getVersionFilesResponseContentType.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-
-        // Test Category Name
-        String testCategory = "testCategory";
-        Response setFileCategoriesResponse = UtilIT.setFileCategories(testFileId1, apiToken, List.of(testCategory));
-        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
-        setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory));
-        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, null, apiToken);
-
-        getVersionFilesResponseCategoryName.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName1))
-                .body("data[1].label", equalTo(testFileName2));
-
-        fileMetadatasCount = getVersionFilesResponseCategoryName.jsonPath().getList("data").size();
-        assertEquals(2, fileMetadatasCount);
-
-        // Test Access Status Restricted
-        Response restrictFileResponse = UtilIT.restrictFile(String.valueOf(testFileId1), true, apiToken);
-        restrictFileResponse.then().assertThat()
-                .statusCode(OK.getStatusCode());
-
-        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, null, apiToken);
-
-        getVersionFilesResponseRestricted.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName1));
-
-        fileMetadatasCount = getVersionFilesResponseRestricted.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-
-        // Test Access Status Embargoed
-        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
-        String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
-
-        // Create embargo for test file 1 (Embargoed and Restricted)
-        Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken);
-
-        createActiveFileEmbargoResponse.then().assertThat()
-                .statusCode(OK.getStatusCode());
-
-        // Create embargo for test file 2 (Embargoed and Public)
-        createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken);
-
-        createActiveFileEmbargoResponse.then().assertThat()
-                .statusCode(OK.getStatusCode());
-
-        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, null, apiToken);
-
-        getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName2));
-
-        fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-
-        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, null, apiToken);
-
-        getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName1));
-
-        fileMetadatasCount = getVersionFilesResponseEmbargoedThenRestricted.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-
-        // Test Access Status Public
-        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, null, apiToken);
-
-        getVersionFilesResponsePublic.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName3))
-                .body("data[1].label", equalTo(testFileName4))
-                .body("data[2].label", equalTo(testFileName5));
-
-        fileMetadatasCount = getVersionFilesResponsePublic.jsonPath().getList("data").size();
-        assertEquals(3, fileMetadatasCount);
-
-        // Test Search Text
-        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, "test_1", null, apiToken);
-
-        getVersionFilesResponseSearchText.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo(testFileName1));
-
-        fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-
-        // Test Tabular Tag Name
-        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
-        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
-
-        // Ensure tabular file is ingested
-        sleep(2000);
-
-        String tabularTagName = "Survey";
-        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(tabularTagName));
-        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        Response getVersionFilesResponseTabularTagName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, tabularTagName, null, null, apiToken);
-
-        getVersionFilesResponseTabularTagName.then().assertThat()
-                .statusCode(OK.getStatusCode())
-                .body("data[0].label", equalTo("test.tab"));
-
-        fileMetadatasCount = getVersionFilesResponseTabularTagName.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
-    }
-
-    @Test
-    public void getVersionFileCounts() throws IOException {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Creating test files
-        String testFileName1 = "test_1.txt";
-        String testFileName2 = "test_2.txt";
-        String testFileName3 = "test_3.png";
-
-        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[50], apiToken);
-        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken);
-        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken);
-
-        // Creating a categorized test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
-        String testCategory = "testCategory";
-        Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory));
-        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Setting embargo for file (Embargo and Public)
-        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
-        String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
-        Response createFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(dataFileId), activeEmbargoDate, apiToken);
-        createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Getting the file counts and assert each count
-        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken);
-
-        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath();
-        LinkedHashMap<String, Integer> responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
-        LinkedHashMap<String, Integer> responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
-        LinkedHashMap<String, Integer> responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
-
-        assertEquals(4, (Integer) responseJsonPath.get("data.total"));
-        assertEquals(2, responseCountPerContentTypeMap.get("image/png"));
-        assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
-        assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
-        assertEquals(3, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString()));
-        assertEquals(1, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString()));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index bfe856a8d18..0a16bca7008 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2,8 +2,6 @@
 
 import io.restassured.RestAssured;
 import io.restassured.response.Response;
-
-import java.util.List;
 import java.util.logging.Logger;
 
 import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism;
@@ -32,12 +30,16 @@
 
 import static jakarta.ws.rs.core.Response.Status.*;
 import org.hamcrest.CoreMatchers;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.startsWith;
+import static org.hamcrest.CoreMatchers.nullValue;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterAll;
 
-import static org.hamcrest.CoreMatchers.*;
-import static org.hamcrest.CoreMatchers.hasItem;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class FilesIT {
 
@@ -2209,137 +2211,4 @@ public void testGetFileDataTables() throws InterruptedException {
         getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(testTabularFileId, randomUserApiToken);
         getFileDataTablesForTabularFileResponse.then().assertThat().statusCode(FORBIDDEN.getStatusCode());
     }
-
-    @Test
-    public void testSetFileCategories() {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Upload test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
-
-        // Set categories
-        String testCategory1 = "testCategory1";
-        String testCategory2 = "testCategory2";
-        List<String> testCategories = List.of(testCategory1, testCategory2);
-        Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, testCategories);
-        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Get file data and check for new categories
-        Response getFileDataResponse = UtilIT.getFileData(dataFileId, apiToken);
-        getFileDataResponse.prettyPrint();
-        getFileDataResponse.then().assertThat()
-                .body("data.categories", hasItem(testCategory1))
-                .body("data.categories", hasItem(testCategory2))
-                .statusCode(OK.getStatusCode());
-    }
-
-    @Test
-    public void testSetFileTabularTags() throws InterruptedException {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Upload tabular file
-        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
-        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
-
-        // Ensure tabular file is ingested
-        sleep(2000);
-
-        // Set tabular tags
-        String testTabularTag1 = "Survey";
-        String testTabularTag2 = "Genomics";
-        List<String> testTabularTags = List.of(testTabularTag1, testTabularTag2);
-        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, testTabularTags);
-        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Get file data and check for new categories
-        Response getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
-        getFileDataResponse.then().assertThat()
-                .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
-                .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
-                .statusCode(OK.getStatusCode());
-
-        // Set invalid tabular tag
-        String testInvalidTabularTag = "Invalid";
-        setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(testInvalidTabularTag));
-        setFileTabularTagsResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
-
-        // Get file data and check categories are unaltered
-        getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
-        getFileDataResponse.then().assertThat()
-                .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
-                .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
-                .statusCode(OK.getStatusCode());
-    }
-
-    @Test
-    public void testGetHasBeenDeleted() {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
-
-        // Upload test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
-
-        // Publish dataverse and dataset
-        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
-        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
-        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Assert that the file has not been deleted
-        Response getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken);
-        getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode());
-        boolean fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
-        assertFalse(fileHasBeenDeleted);
-
-        // Delete test file
-        Response deleteFileInDatasetResponse = UtilIT.deleteFileInDataset(Integer.parseInt(dataFileId), apiToken);
-        deleteFileInDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
-
-        // Assert that the file has been deleted
-        getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken);
-        getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode());
-        fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
-        assertTrue(fileHasBeenDeleted);
-    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 8e333451c8d..f61b392c898 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3276,7 +3276,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3286,21 +3286,6 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (offset != null) {
             requestSpecification = requestSpecification.queryParam("offset", offset);
         }
-        if (contentType != null) {
-            requestSpecification = requestSpecification.queryParam("contentType", contentType);
-        }
-        if (accessStatus != null) {
-            requestSpecification = requestSpecification.queryParam("accessStatus", accessStatus);
-        }
-        if (categoryName != null) {
-            requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
-        }
-        if (tabularTagName != null) {
-            requestSpecification = requestSpecification.queryParam("tabularTagName", tabularTagName);
-        }
-        if (searchText != null) {
-            requestSpecification = requestSpecification.queryParam("searchText", searchText);
-        }
         if (orderCriteria != null) {
             requestSpecification = requestSpecification.queryParam("orderCriteria", orderCriteria);
         }
@@ -3332,75 +3317,9 @@ static Response getFileDataTables(String dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/dataTables");
     }
 
-    static Response getUserFileAccessRequested(String dataFileId, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/access/datafile/" + dataFileId + "/userFileAccessRequested");
-    }
-
     static Response getUserPermissionsOnFile(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/access/datafile/" + dataFileId + "/userPermissions");
     }
-
-    static Response createFileEmbargo(Integer datasetId, Integer fileId, String dateAvailable, String apiToken) {
-        JsonObjectBuilder jsonBuilder = Json.createObjectBuilder();
-        jsonBuilder.add("dateAvailable", dateAvailable);
-        jsonBuilder.add("reason", "This is a test embargo");
-        jsonBuilder.add("fileIds", Json.createArrayBuilder().add(fileId));
-        String jsonString = jsonBuilder.build().toString();
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .body(jsonString)
-                .contentType("application/json")
-                .urlEncodingEnabled(false)
-                .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
-    }
-
-    static Response getVersionFileCounts(Integer datasetId, String version, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
-    }
-
-    static Response setFileCategories(String dataFileId, String apiToken, List<String> categories) {
-        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
-        for (String category : categories) {
-            jsonArrayBuilder.add(category);
-        }
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        jsonObjectBuilder.add("categories", jsonArrayBuilder);
-        String jsonString = jsonObjectBuilder.build().toString();
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .body(jsonString)
-                .post("/api/files/" + dataFileId + "/metadata/categories");
-    }
-
-    static Response setFileTabularTags(String dataFileId, String apiToken, List<String> tabularTags) {
-        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
-        for (String tabularTag : tabularTags) {
-            jsonArrayBuilder.add(tabularTag);
-        }
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        jsonObjectBuilder.add("tabularTags", jsonArrayBuilder);
-        String jsonString = jsonObjectBuilder.build().toString();
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .body(jsonString)
-                .post("/api/files/" + dataFileId + "/metadata/tabularTags");
-    }
-
-    static Response deleteFileInDataset(Integer fileId, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .delete("/api/files/" + fileId);
-    }
-
-    static Response getHasBeenDeleted(String dataFileId, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/files/" + dataFileId + "/hasBeenDeleted");
-    }
 }

From 87393bae0e2f40b41693d7a3cecc64785e81df36 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 15:05:57 +0100
Subject: [PATCH 517/922] Added: docs for filtering by tabular tag in
 getVersionFiles endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index d4e2e4cd178..f78ae62f3ae 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -978,6 +978,14 @@ Usage example:
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?categoryName=Data"
 
+Tabular tag name filtering is also optionally supported. To return files to which the requested tabular tag has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?tabularTagName=Survey"
+
 Content type filtering is also optionally supported. To return files matching the requested content type.
 
 Usage example:

From 9f0b8102904bb663dce8c50203d32663550e2095 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 4 Oct 2023 10:47:54 -0400
Subject: [PATCH 518/922] more tests and docs #9952, #9953, #9957

---
 doc/release-notes/9955-Signposting-updates.md         |  8 +++++++-
 doc/sphinx-guides/source/api/native-api.rst           |  2 +-
 .../edu/harvard/iq/dataverse/api/SignpostingIT.java   | 11 +++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/doc/release-notes/9955-Signposting-updates.md b/doc/release-notes/9955-Signposting-updates.md
index 92168231895..db0e27e51c5 100644
--- a/doc/release-notes/9955-Signposting-updates.md
+++ b/doc/release-notes/9955-Signposting-updates.md
@@ -1 +1,7 @@
-This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification.
\ No newline at end of file
+This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
+
+To fix #9952, we surround the license info with `<` and `>`.
+
+To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
+
+To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index e181a2a5546..bc186720252 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2200,7 +2200,7 @@ Here is an example of a "Link" header:
 
 The URL for linkset information is discoverable under the ``rel="linkset";type="application/linkset+json`` entry in the "Link" header, such as in the example above.
 
-The reponse includes a JSON object conforming to the `Signposting <https://signposting.org>`__ specification.
+The reponse includes a JSON object conforming to the `Signposting <https://signposting.org>`__ specification. As part of this conformance, unlike most Dataverse API responses, the output is not wrapped in a ``{"status":"OK","data":{`` object.
 Signposting is not supported for draft dataset versions.
 
 .. code-block:: bash
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java
index b41f62ae28f..75f514f3398 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/SignpostingIT.java
@@ -80,6 +80,7 @@ public void testSignposting() {
         assertTrue(linkHeader.contains(datasetPid));
         assertTrue(linkHeader.contains("cite-as"));
         assertTrue(linkHeader.contains("describedby"));
+        assertTrue(linkHeader.contains("<http://creativecommons.org/publicdomain/zero/1.0>;rel=\"license\""));
 
         Pattern pattern = Pattern.compile("<([^<]*)> ; rel=\"linkset\";type=\"application\\/linkset\\+json\"");
         Matcher matcher = pattern.matcher(linkHeader);
@@ -101,6 +102,16 @@ public void testSignposting() {
         assertTrue(lso.getString("anchor").indexOf("/dataset.xhtml?persistentId=" + datasetPid) > 0);
         assertTrue(lso.containsKey("describedby"));
 
+        // Test export URL from link header
+        // regex inspired by https://stackoverflow.com/questions/68860255/how-to-match-the-closest-opening-and-closing-brackets
+        Pattern exporterPattern = Pattern.compile("[<\\[][^()\\[\\]]*?exporter=schema.org[^()\\[\\]]*[>\\]]");
+        Matcher exporterMatcher = exporterPattern.matcher(linkHeader);
+        exporterMatcher.find();
+
+        Response exportDataset = UtilIT.exportDataset(datasetPid, "schema.org");
+        exportDataset.prettyPrint();
+        exportDataset.then().assertThat().statusCode(OK.getStatusCode());
+
     }
 
 }

From 5b3630f4bf68a579cbaaf34cd4abf20e278112cb Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 16:22:56 +0100
Subject: [PATCH 519/922] Revert "Revert "Dataset files API extension for
 filters""

This reverts commit 122565a4f1afac9f5fe0d85a3c6f08517eb530c4.
---
 .../9714-files-api-extension-filters.md       |  14 +
 .../9785-files-api-extension-search-text.md   |   3 +
 .../9834-files-api-extension-counts.md        |   6 +
 ...oad-extension-new-file-access-endpoints.md |  14 +
 doc/sphinx-guides/source/api/dataaccess.rst   |  12 +
 doc/sphinx-guides/source/api/native-api.rst   | 165 +++++++++++-
 modules/dataverse-parent/pom.xml              |   3 +
 pom.xml                                       |  14 +
 .../edu/harvard/iq/dataverse/DataFileTag.java |   2 +-
 .../DatasetVersionFilesServiceBean.java       | 241 ++++++++++++++++++
 .../dataverse/DatasetVersionServiceBean.java  |  74 ------
 .../edu/harvard/iq/dataverse/api/Access.java  |  59 +++--
 .../harvard/iq/dataverse/api/Datasets.java    |  50 +++-
 .../edu/harvard/iq/dataverse/api/Files.java   |  90 ++++++-
 .../iq/dataverse/util/json/JsonPrinter.java   |  21 ++
 src/main/java/propertyFiles/Bundle.properties |   1 +
 .../harvard/iq/dataverse/api/AccessIT.java    |  47 ++--
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 240 ++++++++++++++---
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 145 ++++++++++-
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  83 +++++-
 20 files changed, 1110 insertions(+), 174 deletions(-)
 create mode 100644 doc/release-notes/9714-files-api-extension-filters.md
 create mode 100644 doc/release-notes/9785-files-api-extension-search-text.md
 create mode 100644 doc/release-notes/9834-files-api-extension-counts.md
 create mode 100644 doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java

diff --git a/doc/release-notes/9714-files-api-extension-filters.md b/doc/release-notes/9714-files-api-extension-filters.md
new file mode 100644
index 00000000000..034230efe61
--- /dev/null
+++ b/doc/release-notes/9714-files-api-extension-filters.md
@@ -0,0 +1,14 @@
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by:
+
+- Access status: through the `accessStatus` query parameter, which supports the following values:
+
+  - Public
+  - Restricted
+  - EmbargoedThenRestricted
+  - EmbargoedThenPublic
+
+
+- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
+
+
+- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
diff --git a/doc/release-notes/9785-files-api-extension-search-text.md b/doc/release-notes/9785-files-api-extension-search-text.md
new file mode 100644
index 00000000000..fb185e1c7af
--- /dev/null
+++ b/doc/release-notes/9785-files-api-extension-search-text.md
@@ -0,0 +1,3 @@
+The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by search text through the `searchText` query parameter.
+
+The search will be applied to the labels and descriptions of the dataset files.
diff --git a/doc/release-notes/9834-files-api-extension-counts.md b/doc/release-notes/9834-files-api-extension-counts.md
new file mode 100644
index 00000000000..3ec15d8bd36
--- /dev/null
+++ b/doc/release-notes/9834-files-api-extension-counts.md
@@ -0,0 +1,6 @@
+Implemented the following new endpoints:
+
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name). 
+
+
+- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
diff --git a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
new file mode 100644
index 00000000000..f306ae2ab80
--- /dev/null
+++ b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
@@ -0,0 +1,14 @@
+Implemented the following new endpoints:
+
+- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file. 
+
+
+- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
+
+
+In addition, the DataFile API payload has been extended to include the following fields:
+
+- tabularData: Boolean field to know if the DataFile is of tabular type
+
+
+- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner) 
diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 21be702d62b..6edd413b7a5 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -404,6 +404,18 @@ A curl example using an ``id``::
 
     curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests
 
+User Has Requested Access to a File:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``/api/access/datafile/{id}/userFileAccessRequested``
+
+This method returns true or false depending on whether or not the calling user has requested access to a particular file.
+
+A curl example using an ``id``::
+
+    curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userFileAccessRequested"
+
+
 Get User Permissions on a File:
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 5c72937eecb..cf869d338ca 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -970,6 +970,45 @@ This endpoint supports optional pagination, through the ``limit`` and ``offset``
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
 
+Category name filtering is also optionally supported. To return files to which the requested category has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?categoryName=Data"
+
+Content type filtering is also optionally supported. To return files matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?contentType=image/png"
+
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the files that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?searchText=word"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?accessStatus=Public"
+
 Ordering criteria for sorting the results is also optionally supported. In particular, by the following possible values:
 
 * ``NameAZ`` (Default)
@@ -979,14 +1018,42 @@ Ordering criteria for sorting the results is also optionally supported. In parti
 * ``Size``
 * ``Type``
 
-Please note that these values are case sensitive and must be correctly typed for the endpoint to recognize them.
-
 Usage example:
 
 .. code-block:: bash
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?orderCriteria=Newest"
 
+Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+Keep in mind that you can combine all of the above query params depending on the results you are looking for.
+
+Get File Counts in a Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get file counts, for the given dataset and version.
+
+The returned file counts are based on different criteria:
+
+- Total (The total file count)
+- Per content type
+- Per category name
+- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic)
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+  export VERSION=1.0
+
+  curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/files/counts"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts"
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -2832,13 +2899,13 @@ A curl example using an ``ID``
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
 
-  curl "$SERVER_URL/api/files/$ID/downloadCount"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/downloadCount"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/24/downloadCount"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/downloadCount"
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -2848,16 +2915,53 @@ A curl example using a ``PERSISTENT_ID``
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
-  curl "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
 
 If you are interested in download counts for multiple files, see :doc:`/api/metrics`.
 
+File Has Been Deleted
+~~~~~~~~~~~~~~~~~~~~~
+
+Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/hasBeenDeleted"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/hasBeenDeleted"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET  "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000"
+
 Updating File Metadata
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -2907,6 +3011,55 @@ Also note that dataFileTags are not versioned and changes to these will update t
 
 .. _EditingVariableMetadata:
 
+Updating File Metadata Categories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates the categories for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the category names.
+
+Although updating categories can also be done with the previous endpoint, this has been created to be more practical when it is only necessary to update categories and not other metadata fields.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "$SERVER_URL/api/files/$ID/metadata/categories"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "http://demo.dataverse.org/api/files/24/metadata/categories"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+    -F 'jsonData={"categories":["Category1","Category2"]}' \
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000"
+
+Note that if the specified categories do not exist, they will be created.
+
 Editing Variable Level Metadata
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index bfa11af6c70..8e0ff2887df 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -200,6 +200,9 @@
         
         <!-- Container related -->
         <fabric8-dmp.version>0.43.4</fabric8-dmp.version>
+
+        <!-- Persistence -->
+        <querydsl.version>5.0.0</querydsl.version>
     </properties>
     
     <pluginRepositories>
diff --git a/pom.xml b/pom.xml
index e70b723cad5..909e9ee9b80 100644
--- a/pom.xml
+++ b/pom.xml
@@ -252,6 +252,20 @@
             <artifactId>expressly</artifactId>
             <scope>provided</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.querydsl</groupId>
+            <artifactId>querydsl-apt</artifactId>
+            <version>${querydsl.version}</version>
+            <classifier>jakarta</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.querydsl</groupId>
+            <artifactId>querydsl-jpa</artifactId>
+            <version>${querydsl.version}</version>
+            <classifier>jakarta</classifier>
+        </dependency>
         
         <dependency>
             <groupId>commons-io</groupId>
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
index f4f66d3c874..351c4032939 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileTag.java
@@ -58,7 +58,7 @@ public enum TagType {Survey, TimeSeries, Panel, Event, Genomics, Network, Geospa
     
     private static final Map<TagType, String> TagTypeToLabels = new HashMap<>();
     
-    private static final Map<String, TagType> TagLabelToTypes = new HashMap<>();
+    public static final Map<String, TagType> TagLabelToTypes = new HashMap<>();
     
     
     static {
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
new file mode 100644
index 00000000000..6006d937100
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -0,0 +1,241 @@
+package edu.harvard.iq.dataverse;
+
+import edu.harvard.iq.dataverse.QDataFileCategory;
+import edu.harvard.iq.dataverse.QDataFileTag;
+import edu.harvard.iq.dataverse.QDvObject;
+import edu.harvard.iq.dataverse.QEmbargo;
+import edu.harvard.iq.dataverse.QFileMetadata;
+
+import com.querydsl.core.Tuple;
+import com.querydsl.core.types.dsl.BooleanExpression;
+import com.querydsl.core.types.dsl.CaseBuilder;
+import com.querydsl.core.types.dsl.DateExpression;
+import com.querydsl.core.types.dsl.DateTimeExpression;
+
+import com.querydsl.jpa.impl.JPAQuery;
+import com.querydsl.jpa.impl.JPAQueryFactory;
+
+import jakarta.ejb.Stateless;
+import jakarta.inject.Named;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.PersistenceContext;
+
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.time.LocalDate;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
+
+@Stateless
+@Named
+public class DatasetVersionFilesServiceBean implements Serializable {
+
+    @PersistenceContext(unitName = "VDCNet-ejbPU")
+    private EntityManager em;
+
+    private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
+    private final QDvObject dvObject = QDvObject.dvObject;
+    private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
+    private final QDataFileTag dataFileTag = QDataFileTag.dataFileTag;
+
+    /**
+     * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
+     */
+    public enum FileMetadatasOrderCriteria {
+        NameAZ, NameZA, Newest, Oldest, Size, Type
+    }
+
+    /**
+     * Status of the particular DataFile based on active embargoes and restriction state used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
+     */
+    public enum DataFileAccessStatus {
+        Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
+    }
+
+    /**
+     * Given a DatasetVersion, returns its total file metadata count
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return long value of total file metadata count
+     */
+    public long getFileMetadataCount(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).stream().count();
+    }
+
+    /**
+     * Given a DatasetVersion, returns its file metadata count per content type
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return Map<String, Long> of file metadata counts per content type
+     */
+    public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        List<Tuple> contentTypeOccurrences = queryFactory
+                .select(fileMetadata.dataFile.contentType, fileMetadata.count())
+                .from(fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
+                .groupBy(fileMetadata.dataFile.contentType).fetch();
+        Map<String, Long> result = new HashMap<>();
+        for (Tuple occurrence : contentTypeOccurrences) {
+            result.put(occurrence.get(fileMetadata.dataFile.contentType), occurrence.get(fileMetadata.count()));
+        }
+        return result;
+    }
+
+    /**
+     * Given a DatasetVersion, returns its file metadata count per category name
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return Map<String, Long> of file metadata counts per category name
+     */
+    public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        List<Tuple> categoryNameOccurrences = queryFactory
+                .select(dataFileCategory.name, fileMetadata.count())
+                .from(dataFileCategory, fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.fileCategories.contains(dataFileCategory)))
+                .groupBy(dataFileCategory.name).fetch();
+        Map<String, Long> result = new HashMap<>();
+        for (Tuple occurrence : categoryNameOccurrences) {
+            result.put(occurrence.get(dataFileCategory.name), occurrence.get(fileMetadata.count()));
+        }
+        return result;
+    }
+
+    /**
+     * Given a DatasetVersion, returns its file metadata count per DataFileAccessStatus
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @return Map<DataFileAccessStatus, Long> of file metadata counts per DataFileAccessStatus
+     */
+    public Map<DataFileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) {
+        Map<DataFileAccessStatus, Long> allCounts = new HashMap<>();
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Public);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Restricted);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenPublic);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenRestricted);
+        return allCounts;
+    }
+
+    /**
+     * Returns a FileMetadata list of files in the specified DatasetVersion
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param limit          for pagination, can be null
+     * @param offset         for pagination, can be null
+     * @param contentType    for retrieving only files with this content type
+     * @param accessStatus   for retrieving only files with this DataFileAccessStatus
+     * @param categoryName   for retrieving only files categorized with this category name
+     * @param tabularTagName for retrieving only files categorized with this tabular tag name
+     * @param searchText     for retrieving only files that contain the specified text within their labels or descriptions
+     * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
+     * @return a FileMetadata list from the specified DatasetVersion
+     */
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
+        JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
+
+        if (contentType != null) {
+            baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
+        }
+        if (accessStatus != null) {
+            baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
+        }
+        if (categoryName != null) {
+            baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
+        }
+        if (tabularTagName != null) {
+            baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
+        }
+        if (searchText != null && !searchText.isEmpty()) {
+            searchText = searchText.trim().toLowerCase();
+            baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
+        }
+
+        applyOrderCriteriaToGetFileMetadatasQuery(baseQuery, orderCriteria);
+
+        if (limit != null) {
+            baseQuery.limit(limit);
+        }
+        if (offset != null) {
+            baseQuery.offset(offset);
+        }
+
+        return baseQuery.fetch();
+    }
+
+    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<DataFileAccessStatus, Long> totalCounts, DataFileAccessStatus dataFileAccessStatus) {
+        long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus);
+        if (fileMetadataCount > 0) {
+            totalCounts.put(dataFileAccessStatus, fileMetadataCount);
+        }
+    }
+
+    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, DataFileAccessStatus accessStatus) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        return queryFactory
+                .selectFrom(fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus)))
+                .stream().count();
+    }
+
+    private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+        if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) {
+            baseQuery.from(dvObject).where(dvObject.id.eq(fileMetadata.dataFile.id));
+        }
+        return baseQuery;
+    }
+
+    private BooleanExpression createGetFileMetadatasAccessStatusExpression(DataFileAccessStatus accessStatus) {
+        QEmbargo embargo = fileMetadata.dataFile.embargo;
+        BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
+        BooleanExpression inactivelyEmbargoedExpression = embargo.isNull();
+        BooleanExpression accessStatusExpression;
+        switch (accessStatus) {
+            case EmbargoedThenRestricted:
+                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
+                break;
+            case EmbargoedThenPublic:
+                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
+                break;
+            case Restricted:
+                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
+                break;
+            case Public:
+                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
+                break;
+            default:
+                throw new IllegalStateException("Unexpected value: " + accessStatus);
+        }
+        return accessStatusExpression;
+    }
+
+    private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
+        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
+        switch (orderCriteria) {
+            case NameZA:
+                query.orderBy(fileMetadata.label.desc());
+                break;
+            case Newest:
+                query.orderBy(orderByLifetimeExpression.desc());
+                break;
+            case Oldest:
+                query.orderBy(orderByLifetimeExpression.asc());
+                break;
+            case Size:
+                query.orderBy(fileMetadata.dataFile.filesize.asc());
+                break;
+            case Type:
+                query.orderBy(fileMetadata.dataFile.contentType.asc());
+                break;
+            default:
+                query.orderBy(fileMetadata.label.asc());
+                break;
+        }
+    }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 6f087f9eabc..5c43001dcb5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -49,22 +49,6 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
-            + " WHERE fm.datasetVersion.id=:datasetVersionId"
-            + " ORDER BY fm.label";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = dvo.id"
-            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = df.id"
-            + " ORDER BY df.filesize";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = df.id"
-            + " ORDER BY df.contentType";
-
     @EJB
     DatasetServiceBean datasetService;
     
@@ -166,18 +150,6 @@ public DatasetVersion getDatasetVersion(){
         }                
     } // end RetrieveDatasetVersionResponse
 
-    /**
-     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
-     */
-    public enum FileMetadatasOrderCriteria {
-        NameAZ,
-        NameZA,
-        Newest,
-        Oldest,
-        Size,
-        Type
-    }
-
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -1252,50 +1224,4 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
-
-    /**
-     * Returns a FileMetadata list of files in the specified DatasetVersion
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @param limit for pagination, can be null
-     * @param offset for pagination, can be null
-     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
-     * @return a FileMetadata list of the specified DatasetVersion
-     */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
-        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
-                .setParameter("datasetVersionId", datasetVersion.getId());
-        if (limit != null) {
-            query.setMaxResults(limit);
-        }
-        if (offset != null) {
-            query.setFirstResult(offset);
-        }
-        return query.getResultList();
-    }
-
-    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
-        String queryString;
-        switch (orderCriteria) {
-            case NameZA:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
-                break;
-            case Newest:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
-                break;
-            case Oldest:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
-                break;
-            case Size:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
-                break;
-            case Type:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
-                break;
-            default:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
-                break;
-        }
-        return queryString;
-    }
 } // end class
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 1aa3f4ffde6..3c226e68472 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1681,7 +1681,47 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.rejectFailure.noRequest", args));
         }
     }
-    
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userFileAccessRequested")
+    public Response getUserFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        AuthenticatedUser requestAuthenticatedUser;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+            requestAuthenticatedUser = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        boolean fileAccessRequested = false;
+        List<FileAccessRequest> requests = dataFile.getFileAccessRequests();
+        for (FileAccessRequest fileAccessRequest : requests) {
+            if (fileAccessRequest.getRequester().getId().equals(requestAuthenticatedUser.getId())) {
+                fileAccessRequested = true;
+                break;
+            }
+        }
+        return ok(fileAccessRequested);
+    }
+
+    @GET
+    @AuthRequired
+    @Path("/datafile/{id}/userPermissions")
+    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        DataFile dataFile;
+        try {
+            dataFile = findDataFileOrDie(dataFileId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        User requestUser = getRequestUser(crc);
+        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
+        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
+        return ok(jsonObjectBuilder);
+    }
+
     // checkAuthorization is a convenience method; it calls the boolean method
     // isAccessAuthorized(), the actual workhorse, tand throws a 403 exception if not.
     
@@ -1949,21 +1989,4 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin
         }
         return redirectUri;
     }
-
-    @GET
-    @AuthRequired
-    @Path("/datafile/{id}/userPermissions")
-    public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
-        DataFile dataFile;
-        try {
-            dataFile = findDataFileOrDie(dataFileId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        User requestUser = getRequestUser(crc);
-        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
-        jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
-        return ok(jsonObjectBuilder);
-    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 704ec8f1989..b3be55399d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -236,6 +236,9 @@ public class Datasets extends AbstractApiBean {
     @Inject
     PrivateUrlServiceBean privateUrlService;
 
+    @Inject
+    DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
+
     /**
      * Used to consolidate the way we parse and handle dataset versions.
      * @param <T> 
@@ -484,23 +487,56 @@ public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id"
                     : ok(json(dsv));
         }, getRequestUser(crc));
     }
-    
+
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files")
-    public Response getVersionFiles(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @QueryParam("limit") Integer limit, @QueryParam("offset") Integer offset, @QueryParam("orderCriteria") String orderCriteria, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
-        return response( req -> {
+    public Response getVersionFiles(@Context ContainerRequestContext crc,
+                                    @PathParam("id") String datasetId,
+                                    @PathParam("versionId") String versionId,
+                                    @QueryParam("limit") Integer limit,
+                                    @QueryParam("offset") Integer offset,
+                                    @QueryParam("contentType") String contentType,
+                                    @QueryParam("accessStatus") String accessStatus,
+                                    @QueryParam("categoryName") String categoryName,
+                                    @QueryParam("tabularTagName") String tabularTagName,
+                                    @QueryParam("searchText") String searchText,
+                                    @QueryParam("orderCriteria") String orderCriteria,
+                                    @Context UriInfo uriInfo,
+                                    @Context HttpHeaders headers) {
+        return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            DatasetVersionServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
             try {
-                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameAZ;
+                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameAZ;
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            return ok(jsonFileMetadatas(datasetversionService.getFileMetadatas(datasetVersion, limit, offset, fileMetadatasOrderCriteria)));
+            DatasetVersionFilesServiceBean.DataFileAccessStatus dataFileAccessStatus;
+            try {
+                dataFileAccessStatus = accessStatus != null ? DatasetVersionFilesServiceBean.DataFileAccessStatus.valueOf(accessStatus) : null;
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
+            }
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, tabularTagName, searchText, fileMetadatasOrderCriteria)));
         }, getRequestUser(crc));
     }
-    
+
+    @GET
+    @AuthRequired
+    @Path("{id}/versions/{versionId}/files/counts")
+    public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+        return response(req -> {
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+            jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
+            jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion)));
+            jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion)));
+            jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion)));
+            return ok(jsonObjectBuilder);
+        }, getRequestUser(crc));
+    }
+
     @GET
     @AuthRequired
     @Path("{id}/dirindex")
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 82811162d52..8a9abe68d85 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -4,6 +4,7 @@
 import com.google.gson.JsonObject;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
+import edu.harvard.iq.dataverse.DataFileTag;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetLock;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
@@ -54,6 +55,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -63,15 +65,12 @@
 import jakarta.ejb.EJBException;
 import jakarta.inject.Inject;
 import jakarta.json.Json;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonString;
+import jakarta.json.JsonValue;
+import jakarta.json.stream.JsonParsingException;
 import jakarta.servlet.http.HttpServletResponse;
-import jakarta.ws.rs.Consumes;
-import jakarta.ws.rs.DELETE;
-import jakarta.ws.rs.GET;
-import jakarta.ws.rs.POST;
-import jakarta.ws.rs.PUT;
-import jakarta.ws.rs.Path;
-import jakarta.ws.rs.PathParam;
-import jakarta.ws.rs.QueryParam;
+import jakarta.ws.rs.*;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.HttpHeaders;
@@ -112,6 +111,8 @@ public class Files extends AbstractApiBean {
     MakeDataCountLoggingServiceBean mdcLogService;
     @Inject
     GuestbookResponseServiceBean guestbookResponseService;
+    @Inject
+    DataFileServiceBean dataFileServiceBean;
 
     private static final Logger logger = Logger.getLogger(Files.class.getName());
     
@@ -852,18 +853,85 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, @PathPar
         try {
             dataFile = findDataFileOrDie(dataFileId);
         } catch (WrappedResponse e) {
-            return error(Response.Status.NOT_FOUND, "File not found for given id.");
+            return notFound("File not found for given id.");
         }
         if (dataFile.isRestricted() || FileUtil.isActivelyEmbargoed(dataFile)) {
             DataverseRequest dataverseRequest = createDataverseRequest(getRequestUser(crc));
             boolean hasPermissionToDownloadFile = permissionSvc.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
             if (!hasPermissionToDownloadFile) {
-                return error(FORBIDDEN, "Insufficient permissions to access the requested information.");
+                return forbidden("Insufficient permissions to access the requested information.");
             }
         }
         if (!dataFile.isTabularData()) {
-            return error(BAD_REQUEST, "This operation is only available for tabular files.");
+            return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"));
         }
         return ok(jsonDT(dataFile.getDataTables()));
     }
+
+    @POST
+    @AuthRequired
+    @Path("{id}/metadata/categories")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response setFileCategories(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            jakarta.json.JsonObject jsonObject;
+            try (StringReader stringReader = new StringReader(jsonBody)) {
+                jsonObject = Json.createReader(stringReader).readObject();
+                JsonArray requestedCategoriesJson = jsonObject.getJsonArray("categories");
+                FileMetadata fileMetadata = dataFile.getFileMetadata();
+                for (JsonValue jsonValue : requestedCategoriesJson) {
+                    JsonString jsonString = (JsonString) jsonValue;
+                    fileMetadata.addCategoryByName(jsonString.getString());
+                }
+                execCommand(new UpdateDatasetVersionCommand(fileMetadata.getDataFile().getOwner(), req));
+                return ok("Categories of file " + dataFileId + " updated.");
+            } catch (JsonParsingException jpe) {
+                return badRequest("Error parsing Json: " + jpe.getMessage());
+            }
+        }, getRequestUser(crc));
+    }
+
+    @POST
+    @AuthRequired
+    @Path("{id}/metadata/tabularTags")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response setFileTabularTags(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            if (!dataFile.isTabularData()) {
+                return badRequest(BundleUtil.getStringFromBundle("files.api.only.tabular.supported"));
+            }
+            jakarta.json.JsonObject jsonObject;
+            try (StringReader stringReader = new StringReader(jsonBody)) {
+                jsonObject = Json.createReader(stringReader).readObject();
+                JsonArray requestedTabularTagsJson = jsonObject.getJsonArray("tabularTags");
+                for (JsonValue jsonValue : requestedTabularTagsJson) {
+                    JsonString jsonString = (JsonString) jsonValue;
+                    DataFileTag tag = new DataFileTag();
+                    try {
+                        tag.setTypeByLabel(jsonString.getString());
+                    } catch (IllegalArgumentException iax) {
+                        return badRequest(iax.getMessage());
+                    }
+                    tag.setDataFile(dataFile);
+                    dataFile.addTag(tag);
+                }
+                execCommand(new UpdateDatasetVersionCommand(dataFile.getOwner(), req));
+                return ok("Tabular tags of file " + dataFileId + " updated.");
+            } catch (JsonParsingException jpe) {
+                return badRequest("Error parsing Json: " + jpe.getMessage());
+            }
+        }, getRequestUser(crc));
+    }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/hasBeenDeleted")
+    public Response getHasBeenDeleted(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) {
+        return response(req -> {
+            DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId)));
+            return ok(dataFileServiceBean.hasBeenDeleted(dataFile));
+        }, getRequestUser(crc));
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index a7aa36f179e..1fed0b233e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -672,9 +672,14 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo
                 //---------------------------------------------
                 .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue()))
                 .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue()))
+                .add("tabularData", df.isTabularData())
                 .add("tabularTags", getTabularFileTags(df))
                 .add("creationDate", df.getCreateDateFormattedYYYYMMDD())
                 .add("publicationDate",  df.getPublicationDateFormattedYYYYMMDD());
+        Dataset dfOwner = df.getOwner();
+        if (dfOwner != null) {
+            builder.add("fileAccessRequest", dfOwner.isFileAccessRequest());
+        }
         /*
          * The restricted state was not included prior to #9175 so to avoid backward
          * incompatability, it is now only added when generating json for the
@@ -1095,6 +1100,22 @@ public Set<Collector.Characteristics> characteristics() {
         };
     }
 
+    public static JsonObjectBuilder json(Map<String, Long> map) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        for (Map.Entry<String, Long> mapEntry : map.entrySet()) {
+            jsonObjectBuilder.add(mapEntry.getKey(), mapEntry.getValue());
+        }
+        return jsonObjectBuilder;
+    }
+
+    public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> map) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        for (Map.Entry<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> mapEntry : map.entrySet()) {
+            jsonObjectBuilder.add(mapEntry.getKey().toString(), mapEntry.getValue());
+        }
+        return jsonObjectBuilder;
+    }
+
     public static Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder> toJsonArray() {
         return new Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder>() {
 
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index ac725caf1b2..7b4befcca36 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2620,6 +2620,7 @@ admin.api.deleteUser.success=Authenticated User {0} deleted.
 #Files.java
 files.api.metadata.update.duplicateFile=Filename already exists at {0}
 files.api.no.draft=No draft available for this file
+files.api.only.tabular.supported=This operation is only available for tabular files.
 
 #Datasets.java
 datasets.api.updatePIDMetadata.failure.dataset.must.be.released=Modify Registration Metadata must be run on a published dataset.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index b1beddd893f..416caa68566 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -26,11 +26,9 @@
 
 import static jakarta.ws.rs.core.Response.Status.*;
 import static org.hamcrest.MatcherAssert.*;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
+import static org.junit.jupiter.api.Assertions.*;
 
 /**
  *
@@ -632,28 +630,37 @@ public void testZipUploadAndDownload() throws IOException {
     }
 
     @Test
-    public void testGetUserPermissionsOnFile() {
-        Response createUser = UtilIT.createRandomUser();
-        createUser.then().assertThat().statusCode(OK.getStatusCode());
-        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+    public void testGetUserFileAccessRequested() {
+        // Create new user
+        Response createUserResponse = UtilIT.createRandomUser();
+        createUserResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String newUserApiToken = UtilIT.getApiTokenFromResponse(createUserResponse);
 
-        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
-        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+        String dataFileId = Integer.toString(tabFile3IdRestricted);
 
-        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
-        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        // Call with new user and unrequested access file
+        Response getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken);
+        getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Upload test file
-        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
-        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
-        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data");
+        assertFalse(userFileAccessRequested);
+
+        // Request file access for the new user
+        Response requestFileAccessResponse = UtilIT.requestFileAccess(dataFileId, newUserApiToken);
+        requestFileAccessResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Assert user permissions on file
-        int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id");
-        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(testFileId), apiToken);
+        // Call with new user and requested access file
+        getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken);
+        getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode());
 
+        userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data");
+        assertTrue(userFileAccessRequested);
+    }
+
+    @Test
+    public void testGetUserPermissionsOnFile() {
+        // Call with valid file id
+        Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(basicFileId), apiToken);
         getUserPermissionsOnFileResponse.then().assertThat().statusCode(OK.getStatusCode());
         boolean canDownloadFile = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canDownloadFile");
         assertTrue(canDownloadFile);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 3b6d4d1ecdf..b9f09cc7c07 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -1,6 +1,6 @@
 package edu.harvard.iq.dataverse.api;
 
-import edu.harvard.iq.dataverse.DatasetVersionServiceBean;
+import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
 import io.restassured.RestAssured;
 
 import static io.restassured.RestAssured.given;
@@ -9,6 +9,9 @@
 import io.restassured.http.ContentType;
 import io.restassured.response.Response;
 
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
 import java.util.logging.Logger;
 
 import org.junit.jupiter.api.AfterAll;
@@ -17,8 +20,6 @@
 import org.skyscreamer.jsonassert.JSONAssert;
 import org.junit.jupiter.api.Disabled;
 
-import java.util.List;
-import java.util.Map;
 import jakarta.json.JsonObject;
 
 import static jakarta.ws.rs.core.Response.Status.CREATED;
@@ -39,8 +40,6 @@
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
-import java.util.UUID;
-
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 
@@ -67,8 +66,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.HashMap;
+
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObjectBuilder;
@@ -77,6 +75,7 @@
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 
+import static java.lang.Thread.sleep;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import org.hamcrest.CoreMatchers;
@@ -117,7 +116,9 @@ public static void setUpClass() {
         Response removeAnonymizedFieldTypeNames = UtilIT.deleteSetting(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
         removeAnonymizedFieldTypeNames.then().assertThat()
                 .statusCode(200);
-        
+
+        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+
         /* With Dual mode, we can no longer mess with upload methods since native is now required for anything to work
                
         Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl);
@@ -144,7 +145,9 @@ public static void afterClass() {
         Response removeAnonymizedFieldTypeNames = UtilIT.deleteSetting(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
         removeAnonymizedFieldTypeNames.then().assertThat()
                 .statusCode(200);
-        
+
+        UtilIT.deleteSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths);
+
         /* See above
         Response removeDcmUrl = UtilIT.deleteSetting(SettingsServiceBean.Key.DataCaptureModuleUrl);
         removeDcmUrl.then().assertThat()
@@ -3264,8 +3267,7 @@ public void getDatasetVersionCitation() {
                 .body("data.message", containsString("DRAFT VERSION"));
     }
 
-    @Test
-    public void getVersionFiles() throws IOException {
+    public void getVersionFiles() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3297,39 +3299,42 @@ public void getVersionFiles() throws IOException {
         int testPageSize = 2;
 
         // Test page 1
-        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, apiToken);
-
-        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(testPageSize, fileMetadatasCount);
+        Response getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, null, null, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName1))
                 .body("data[1].label", equalTo(testFileName2));
 
-        // Test page 2
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, apiToken);
-
-        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        int fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
         assertEquals(testPageSize, fileMetadatasCount);
 
+        String testFileId1 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id");
+        String testFileId2 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[1].dataFile.id");
+
+        // Test page 2
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize, null, null, null, null, null, null, apiToken);
+
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName3))
                 .body("data[1].label", equalTo(testFileName4));
 
-        // Test page 3 (last)
-        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, apiToken);
-
         fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
-        assertEquals(1, fileMetadatasCount);
+        assertEquals(testPageSize, fileMetadatasCount);
+
+        // Test page 3 (last)
+        getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, null, apiToken);
 
         getVersionFilesResponsePaginated.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 .body("data[0].label", equalTo(testFileName5));
 
+        fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3340,7 +3345,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3351,7 +3356,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3362,7 +3367,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3373,7 +3378,7 @@ public void getVersionFiles() throws IOException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, DatasetVersionServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3385,9 +3390,186 @@ public void getVersionFiles() throws IOException {
 
         // Test invalid order criteria
         String invalidOrderCriteria = "invalidOrderCriteria";
-        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, invalidOrderCriteria, apiToken);
+        Response getVersionFilesResponseInvalidOrderCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, invalidOrderCriteria, apiToken);
         getVersionFilesResponseInvalidOrderCriteria.then().assertThat()
                 .statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid order criteria: " + invalidOrderCriteria));
+
+        // Test Content Type
+        Response getVersionFilesResponseContentType = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, null, null, null, null, apiToken);
+
+        getVersionFilesResponseContentType.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName5));
+
+        fileMetadatasCount = getVersionFilesResponseContentType.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        // Test Category Name
+        String testCategory = "testCategory";
+        Response setFileCategoriesResponse = UtilIT.setFileCategories(testFileId1, apiToken, List.of(testCategory));
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+        setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory));
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, null, apiToken);
+
+        getVersionFilesResponseCategoryName.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1))
+                .body("data[1].label", equalTo(testFileName2));
+
+        fileMetadatasCount = getVersionFilesResponseCategoryName.jsonPath().getList("data").size();
+        assertEquals(2, fileMetadatasCount);
+
+        // Test Access Status Restricted
+        Response restrictFileResponse = UtilIT.restrictFile(String.valueOf(testFileId1), true, apiToken);
+        restrictFileResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, null, apiToken);
+
+        getVersionFilesResponseRestricted.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1));
+
+        fileMetadatasCount = getVersionFilesResponseRestricted.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        // Test Access Status Embargoed
+        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
+        String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+
+        // Create embargo for test file 1 (Embargoed and Restricted)
+        Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken);
+
+        createActiveFileEmbargoResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        // Create embargo for test file 2 (Embargoed and Public)
+        createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken);
+
+        createActiveFileEmbargoResponse.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, null, apiToken);
+
+        getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName2));
+
+        fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, null, apiToken);
+
+        getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1));
+
+        fileMetadatasCount = getVersionFilesResponseEmbargoedThenRestricted.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        // Test Access Status Public
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, null, apiToken);
+
+        getVersionFilesResponsePublic.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName3))
+                .body("data[1].label", equalTo(testFileName4))
+                .body("data[2].label", equalTo(testFileName5));
+
+        fileMetadatasCount = getVersionFilesResponsePublic.jsonPath().getList("data").size();
+        assertEquals(3, fileMetadatasCount);
+
+        // Test Search Text
+        Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, "test_1", null, apiToken);
+
+        getVersionFilesResponseSearchText.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo(testFileName1));
+
+        fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+
+        // Test Tabular Tag Name
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Ensure tabular file is ingested
+        sleep(2000);
+
+        String tabularTagName = "Survey";
+        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(tabularTagName));
+        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response getVersionFilesResponseTabularTagName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, tabularTagName, null, null, apiToken);
+
+        getVersionFilesResponseTabularTagName.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data[0].label", equalTo("test.tab"));
+
+        fileMetadatasCount = getVersionFilesResponseTabularTagName.jsonPath().getList("data").size();
+        assertEquals(1, fileMetadatasCount);
+    }
+
+    @Test
+    public void getVersionFileCounts() throws IOException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Creating test files
+        String testFileName1 = "test_1.txt";
+        String testFileName2 = "test_2.txt";
+        String testFileName3 = "test_3.png";
+
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[50], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken);
+        UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken);
+
+        // Creating a categorized test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+        String testCategory = "testCategory";
+        Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory));
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Setting embargo for file (Embargo and Public)
+        UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12");
+        String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+        Response createFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(dataFileId), activeEmbargoDate, apiToken);
+        createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Getting the file counts and assert each count
+        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        LinkedHashMap<String, Integer> responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        LinkedHashMap<String, Integer> responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        LinkedHashMap<String, Integer> responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(4, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(2, responseCountPerContentTypeMap.get("image/png"));
+        assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
+        assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(3, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString()));
+        assertEquals(1, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString()));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 0a16bca7008..bfe856a8d18 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2,6 +2,8 @@
 
 import io.restassured.RestAssured;
 import io.restassured.response.Response;
+
+import java.util.List;
 import java.util.logging.Logger;
 
 import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism;
@@ -30,16 +32,12 @@
 
 import static jakarta.ws.rs.core.Response.Status.*;
 import org.hamcrest.CoreMatchers;
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.startsWith;
-import static org.hamcrest.CoreMatchers.nullValue;
 import org.hamcrest.Matchers;
 import org.junit.jupiter.api.AfterAll;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.hamcrest.CoreMatchers.*;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.junit.jupiter.api.Assertions.*;
 
 public class FilesIT {
 
@@ -2211,4 +2209,137 @@ public void testGetFileDataTables() throws InterruptedException {
         getFileDataTablesForTabularFileResponse = UtilIT.getFileDataTables(testTabularFileId, randomUserApiToken);
         getFileDataTablesForTabularFileResponse.then().assertThat().statusCode(FORBIDDEN.getStatusCode());
     }
+
+    @Test
+    public void testSetFileCategories() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Set categories
+        String testCategory1 = "testCategory1";
+        String testCategory2 = "testCategory2";
+        List<String> testCategories = List.of(testCategory1, testCategory2);
+        Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, testCategories);
+        setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get file data and check for new categories
+        Response getFileDataResponse = UtilIT.getFileData(dataFileId, apiToken);
+        getFileDataResponse.prettyPrint();
+        getFileDataResponse.then().assertThat()
+                .body("data.categories", hasItem(testCategory1))
+                .body("data.categories", hasItem(testCategory2))
+                .statusCode(OK.getStatusCode());
+    }
+
+    @Test
+    public void testSetFileTabularTags() throws InterruptedException {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload tabular file
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Ensure tabular file is ingested
+        sleep(2000);
+
+        // Set tabular tags
+        String testTabularTag1 = "Survey";
+        String testTabularTag2 = "Genomics";
+        List<String> testTabularTags = List.of(testTabularTag1, testTabularTag2);
+        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, testTabularTags);
+        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Get file data and check for new categories
+        Response getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
+        getFileDataResponse.then().assertThat()
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
+                .statusCode(OK.getStatusCode());
+
+        // Set invalid tabular tag
+        String testInvalidTabularTag = "Invalid";
+        setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(testInvalidTabularTag));
+        setFileTabularTagsResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+
+        // Get file data and check categories are unaltered
+        getFileDataResponse = UtilIT.getFileData(tabularFileId, apiToken);
+        getFileDataResponse.then().assertThat()
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag1))
+                .body("data.dataFile.tabularTags", hasItem(testTabularTag2))
+                .statusCode(OK.getStatusCode());
+    }
+
+    @Test
+    public void testGetHasBeenDeleted() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Upload test file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Publish dataverse and dataset
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert that the file has not been deleted
+        Response getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken);
+        getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
+        assertFalse(fileHasBeenDeleted);
+
+        // Delete test file
+        Response deleteFileInDatasetResponse = UtilIT.deleteFileInDataset(Integer.parseInt(dataFileId), apiToken);
+        deleteFileInDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Assert that the file has been deleted
+        getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken);
+        getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode());
+        fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
+        assertTrue(fileHasBeenDeleted);
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index f61b392c898..8e333451c8d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3276,7 +3276,7 @@ static Response getDatasetVersionCitation(Integer datasetId, String version, Str
         return response;
     }
 
-    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String orderCriteria, String apiToken) {
+    static Response getVersionFiles(Integer datasetId, String version, Integer limit, Integer offset, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, String orderCriteria, String apiToken) {
         RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json");
@@ -3286,6 +3286,21 @@ static Response getVersionFiles(Integer datasetId, String version, Integer limit
         if (offset != null) {
             requestSpecification = requestSpecification.queryParam("offset", offset);
         }
+        if (contentType != null) {
+            requestSpecification = requestSpecification.queryParam("contentType", contentType);
+        }
+        if (accessStatus != null) {
+            requestSpecification = requestSpecification.queryParam("accessStatus", accessStatus);
+        }
+        if (categoryName != null) {
+            requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
+        }
+        if (tabularTagName != null) {
+            requestSpecification = requestSpecification.queryParam("tabularTagName", tabularTagName);
+        }
+        if (searchText != null) {
+            requestSpecification = requestSpecification.queryParam("searchText", searchText);
+        }
         if (orderCriteria != null) {
             requestSpecification = requestSpecification.queryParam("orderCriteria", orderCriteria);
         }
@@ -3317,9 +3332,75 @@ static Response getFileDataTables(String dataFileId, String apiToken) {
                 .get("/api/files/" + dataFileId + "/dataTables");
     }
 
+    static Response getUserFileAccessRequested(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/access/datafile/" + dataFileId + "/userFileAccessRequested");
+    }
+
     static Response getUserPermissionsOnFile(String dataFileId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .get("/api/access/datafile/" + dataFileId + "/userPermissions");
     }
+
+    static Response createFileEmbargo(Integer datasetId, Integer fileId, String dateAvailable, String apiToken) {
+        JsonObjectBuilder jsonBuilder = Json.createObjectBuilder();
+        jsonBuilder.add("dateAvailable", dateAvailable);
+        jsonBuilder.add("reason", "This is a test embargo");
+        jsonBuilder.add("fileIds", Json.createArrayBuilder().add(fileId));
+        String jsonString = jsonBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .contentType("application/json")
+                .urlEncodingEnabled(false)
+                .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
+    }
+
+    static Response getVersionFileCounts(Integer datasetId, String version, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
+    }
+
+    static Response setFileCategories(String dataFileId, String apiToken, List<String> categories) {
+        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
+        for (String category : categories) {
+            jsonArrayBuilder.add(category);
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("categories", jsonArrayBuilder);
+        String jsonString = jsonObjectBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .post("/api/files/" + dataFileId + "/metadata/categories");
+    }
+
+    static Response setFileTabularTags(String dataFileId, String apiToken, List<String> tabularTags) {
+        JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
+        for (String tabularTag : tabularTags) {
+            jsonArrayBuilder.add(tabularTag);
+        }
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("tabularTags", jsonArrayBuilder);
+        String jsonString = jsonObjectBuilder.build().toString();
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(jsonString)
+                .post("/api/files/" + dataFileId + "/metadata/tabularTags");
+    }
+
+    static Response deleteFileInDataset(Integer fileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .delete("/api/files/" + fileId);
+    }
+
+    static Response getHasBeenDeleted(String dataFileId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/files/" + dataFileId + "/hasBeenDeleted");
+    }
 }

From 440d02030d12852d61a38c458d6d21d1f9f92bfd Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 16:40:29 +0100
Subject: [PATCH 520/922] Added: release notes for PR #9972

---
 doc/release-notes/9972-files-api-filter-by-tabular-tags.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/9972-files-api-filter-by-tabular-tags.md

diff --git a/doc/release-notes/9972-files-api-filter-by-tabular-tags.md b/doc/release-notes/9972-files-api-filter-by-tabular-tags.md
new file mode 100644
index 00000000000..9c3fced1741
--- /dev/null
+++ b/doc/release-notes/9972-files-api-filter-by-tabular-tags.md
@@ -0,0 +1,3 @@
+- New query parameter `tabularTagName` added to the getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) to return files to which the particular tabular tag has been added.
+
+- New endpoint to set tabular file tags via API: /api/files/{id}/metadata/tabularTags.

From eb56c502dc2179754fa144bf0f354da444612ea9 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 26 Sep 2023 16:26:21 -0400
Subject: [PATCH 521/922] allow fast xhtml edits in Docker #9590

---
 docker-compose-dev.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 694f2046ca8..930bb1230f5 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -18,6 +18,7 @@ services:
       - DATAVERSE_AUTH_OIDC_CLIENT_ID=test
       - DATAVERSE_AUTH_OIDC_CLIENT_SECRET=94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8
       - DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL=http://keycloak.mydomain.com:8090/realms/test
+      - DATAVERSE_JSF_REFRESH_PERIOD=1
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4848:4848" # HTTP (Payara Admin Console)
@@ -31,6 +32,9 @@ services:
     volumes:
       - ./docker-dev-volumes/app/data:/dv
       - ./docker-dev-volumes/app/secrets:/secrets
+      # Uncomment for changes to xhtml in Netbeans to be deployed immediately.
+      # Replace 6.0 with the current version.
+      # - ./target/dataverse-6.0:/opt/payara/deployments/dataverse
     tmpfs:
       - /dumps:mode=770,size=2052M,uid=1000,gid=1000
       - /tmp:mode=770,size=2052M,uid=1000,gid=1000

From db1410e8c14c73becf7042b4d5ebca6c0c9ef8bc Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 4 Oct 2023 17:15:01 +0100
Subject: [PATCH 522/922] Added: non-tabular error test case to
 setFileTabularTags IT

---
 .../java/edu/harvard/iq/dataverse/api/FilesIT.java     | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index af678905fe1..af020215f5d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2304,6 +2304,16 @@ public void testSetFileTabularTags() throws InterruptedException {
 
         actualTabularTagsCount = getFileDataResponse.jsonPath().getList("data.dataFile.tabularTags").size();
         assertEquals(2, actualTabularTagsCount);
+
+        // Should receive an error when calling the endpoint for a non-tabular file
+        String pathToTestFile = "src/test/resources/images/coffeeshop.png";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String nonTabularFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        setFileTabularTagsResponse = UtilIT.setFileTabularTags(nonTabularFileId, apiToken, List.of(testInvalidTabularTag));
+        setFileTabularTagsResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
 
     @Test

From e2fa09680472fdcda617efb430d0b14ccce70ab9 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 4 Oct 2023 12:15:18 -0400
Subject: [PATCH 523/922] instead of Netbeans, be generic about IDEs/tools
 #9590

---
 docker-compose-dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 930bb1230f5..d4564ab1335 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -32,7 +32,7 @@ services:
     volumes:
       - ./docker-dev-volumes/app/data:/dv
       - ./docker-dev-volumes/app/secrets:/secrets
-      # Uncomment for changes to xhtml in Netbeans to be deployed immediately.
+      # Uncomment for changes to xhtml to be deployed immediately (if supported your IDE or toolchain).
       # Replace 6.0 with the current version.
       # - ./target/dataverse-6.0:/opt/payara/deployments/dataverse
     tmpfs:

From 9d7108b7ed2759dade7f6d1b67a1ec1ab541cc95 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Wed, 4 Oct 2023 23:59:11 +0200
Subject: [PATCH 524/922] chore(build): use stable Rewrite releases for Jakarta
 EE 10 #8305

---
 modules/dataverse-parent/pom.xml | 3 ++-
 pom.xml                          | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index bfa11af6c70..b6d846b49bc 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -426,6 +426,7 @@
             <url>https://artifacts.unidata.ucar.edu/repository/unidata-all/</url>
         </repository>
         <!-- Uncomment when using snapshot releases from Maven Central -->
+        <!--
         <repository>
             <id>oss-sonatype</id>
             <name>oss-sonatype</name>
@@ -446,7 +447,7 @@
                 <enabled>true</enabled>
             </snapshots>
         </repository>
-        <!-- -->
+        -->
     </repositories>
     
     <profiles>
diff --git a/pom.xml b/pom.xml
index e70b723cad5..5536bcccb05 100644
--- a/pom.xml
+++ b/pom.xml
@@ -359,12 +359,12 @@
         <dependency>
             <groupId>org.ocpsoft.rewrite</groupId>
             <artifactId>rewrite-servlet</artifactId>
-            <version>6.0.0-SNAPSHOT</version>
+            <version>10.0.0.Final</version>
         </dependency>
         <dependency>
             <groupId>org.ocpsoft.rewrite</groupId>
             <artifactId>rewrite-config-prettyfaces</artifactId>
-            <version>6.0.0-SNAPSHOT</version>
+            <version>10.0.0.Final</version>
         </dependency>
         <dependency>
             <groupId>edu.ucsb.nceas</groupId>

From 365b18cf2ba70d2ac192a04475ede920eb11aa6c Mon Sep 17 00:00:00 2001
From: Lehebax <lehebax724@ktasy.com>
Date: Thu, 5 Oct 2023 12:07:41 +0530
Subject: [PATCH 525/922] Fixed the equals() method check

---
 .../java/edu/harvard/iq/dataverse/DatasetFieldDefaultValue.java | 2 +-
 src/main/java/edu/harvard/iq/dataverse/DataverseContact.java    | 2 +-
 src/main/java/edu/harvard/iq/dataverse/DataverseFacet.java      | 2 +-
 .../edu/harvard/iq/dataverse/DataverseFeaturedDataverse.java    | 2 +-
 src/main/java/edu/harvard/iq/dataverse/DataverseTheme.java      | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValue.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValue.java
index 7746099818e..8ac98500890 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValue.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValue.java
@@ -113,7 +113,7 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object object) {
-        if (!(object instanceof DatasetField)) {
+        if (!(object instanceof DatasetFieldDefaultValue)) {
             return false;
         }
         DatasetFieldDefaultValue other = (DatasetFieldDefaultValue) object;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseContact.java b/src/main/java/edu/harvard/iq/dataverse/DataverseContact.java
index d77767985eb..9f86a03639a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseContact.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseContact.java
@@ -99,7 +99,7 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object object) {
-        if (!(object instanceof DatasetFieldType)) {
+        if (!(object instanceof DataverseContact)) {
             return false;
         }
         DataverseContact other = (DataverseContact) object;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFacet.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFacet.java
index 768c2308e50..83a2d8fdb8f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseFacet.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFacet.java
@@ -93,7 +93,7 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object object) {
-        if (!(object instanceof DatasetFieldType)) {
+        if (!(object instanceof DataverseFacet)) {
             return false;
         }
         DataverseFacet other = (DataverseFacet) object;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverse.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverse.java
index 39ad6ca9520..d30d94cd034 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverse.java
@@ -85,7 +85,7 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object object) {
-        if (!(object instanceof DatasetFieldType)) {
+        if (!(object instanceof DataverseFeaturedDataverse)) {
             return false;
         }
         DataverseFeaturedDataverse other = (DataverseFeaturedDataverse) object;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseTheme.java b/src/main/java/edu/harvard/iq/dataverse/DataverseTheme.java
index 539669328a7..7f57d16b95a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseTheme.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseTheme.java
@@ -181,7 +181,7 @@ public int hashCode() {
 
     @Override
     public boolean equals(Object object) {
-        if (!(object instanceof DatasetFieldType)) {
+        if (!(object instanceof DataverseTheme)) {
             return false;
         }
         DataverseTheme other = (DataverseTheme) object;

From f42587e1e33af7170185cc24ce382cb402d72533 Mon Sep 17 00:00:00 2001
From: Lehebax <lehebax724@ktasy.com>
Date: Thu, 5 Oct 2023 19:18:37 +0530
Subject: [PATCH 526/922] Added unit tests for the fixed equals() methods

---
 .../DatasetFieldDefaultValueTest.java         | 47 +++++++++++++++++++
 .../iq/dataverse/DataverseContactTest.java    | 47 +++++++++++++++++++
 .../iq/dataverse/DataverseFacetTest.java      | 47 +++++++++++++++++++
 .../DataverseFeaturedDataverseTest.java       | 47 +++++++++++++++++++
 .../iq/dataverse/DataverseThemeTest.java      | 47 +++++++++++++++++++
 5 files changed, 235 insertions(+)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValueTest.java
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/DataverseContactTest.java
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/DataverseFacetTest.java
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverseTest.java
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/DataverseThemeTest.java

diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValueTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValueTest.java
new file mode 100644
index 00000000000..999fadaae06
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldDefaultValueTest.java
@@ -0,0 +1,47 @@
+package edu.harvard.iq.dataverse;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import org.junit.jupiter.api.BeforeEach;
+
+public class DatasetFieldDefaultValueTest {
+    private DatasetFieldDefaultValue dataverseContact;
+
+    @BeforeEach
+    public void before() {
+        this.dataverseContact = new DatasetFieldDefaultValue();
+        this.dataverseContact.setId(1L);
+    }
+
+    @Test
+    public void testEqualsWithNull() {
+        assertFalse(this.dataverseContact.equals(null));
+    }
+
+    @Test
+    public void testEqualsWithDifferentClass() {
+        DatasetField datasetField = new DatasetField();
+
+        assertFalse(this.dataverseContact.equals(datasetField));
+    }
+
+    @Test
+    public void testEqualsWithSameClassSameId() {
+        DatasetFieldDefaultValue dataverseContact1 = new DatasetFieldDefaultValue();
+        dataverseContact1.setId(1L);
+
+        assertTrue(this.dataverseContact.equals(dataverseContact1));
+    }
+
+    @Test
+    public void testEqualsWithSameClassDifferentId() {
+        DatasetFieldDefaultValue dataverseContact1 = new DatasetFieldDefaultValue();
+        dataverseContact1.setId(2L);
+
+        assertFalse(this.dataverseContact.equals(dataverseContact1));
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/DataverseContactTest.java b/src/test/java/edu/harvard/iq/dataverse/DataverseContactTest.java
new file mode 100644
index 00000000000..2abb10a485d
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/DataverseContactTest.java
@@ -0,0 +1,47 @@
+package edu.harvard.iq.dataverse;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import org.junit.jupiter.api.BeforeEach;
+
+public class DataverseContactTest {
+    private DataverseContact dataverseContact;
+
+    @BeforeEach
+    public void before() {
+        this.dataverseContact = new DataverseContact();
+        this.dataverseContact.setId(1L);
+    }
+
+    @Test
+    public void testEqualsWithNull() {
+        assertFalse(this.dataverseContact.equals(null));
+    }
+
+    @Test
+    public void testEqualsWithDifferentClass() {
+        DatasetFieldType datasetFieldType = new DatasetFieldType();
+
+        assertFalse(this.dataverseContact.equals(datasetFieldType));
+    }
+
+    @Test
+    public void testEqualsWithSameClassSameId() {
+        DataverseContact dataverseContact1 = new DataverseContact();
+        dataverseContact1.setId(1L);
+
+        assertTrue(this.dataverseContact.equals(dataverseContact1));
+    }
+
+    @Test
+    public void testEqualsWithSameClassDifferentId() {
+        DataverseContact dataverseContact1 = new DataverseContact();
+        dataverseContact1.setId(2L);
+
+        assertFalse(this.dataverseContact.equals(dataverseContact1));
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/DataverseFacetTest.java b/src/test/java/edu/harvard/iq/dataverse/DataverseFacetTest.java
new file mode 100644
index 00000000000..7ae50439c10
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/DataverseFacetTest.java
@@ -0,0 +1,47 @@
+package edu.harvard.iq.dataverse;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import org.junit.jupiter.api.BeforeEach;
+
+public class DataverseFacetTest {
+    private DataverseFacet dataverseFacet;
+
+    @BeforeEach
+    public void before() {
+        this.dataverseFacet = new DataverseFacet();
+        this.dataverseFacet.setId(1L);
+    }
+
+    @Test
+    public void testEqualsWithNull() {
+        assertFalse(this.dataverseFacet.equals(null));
+    }
+
+    @Test
+    public void testEqualsWithDifferentClass() {
+        DatasetFieldType datasetFieldType = new DatasetFieldType();
+
+        assertFalse(this.dataverseFacet.equals(datasetFieldType));
+    }
+
+    @Test
+    public void testEqualsWithSameClassSameId() {
+        DataverseFacet dataverseFacet1 = new DataverseFacet();
+        dataverseFacet1.setId(1L);
+
+        assertTrue(this.dataverseFacet.equals(dataverseFacet1));
+    }
+
+    @Test
+    public void testEqualsWithSameClassDifferentId() {
+        DataverseFacet dataverseFacet1 = new DataverseFacet();
+        dataverseFacet1.setId(2L);
+
+        assertFalse(this.dataverseFacet.equals(dataverseFacet1));
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverseTest.java b/src/test/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverseTest.java
new file mode 100644
index 00000000000..b024dc3bfd3
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/DataverseFeaturedDataverseTest.java
@@ -0,0 +1,47 @@
+package edu.harvard.iq.dataverse;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import org.junit.jupiter.api.BeforeEach;
+
+public class DataverseFeaturedDataverseTest {
+    private DataverseFeaturedDataverse dataverseFeaturedDataverse;
+
+    @BeforeEach
+    public void before() {
+        this.dataverseFeaturedDataverse = new DataverseFeaturedDataverse();
+        this.dataverseFeaturedDataverse.setId(1L);
+    }
+
+    @Test
+    public void testEqualsWithNull() {
+        assertFalse(this.dataverseFeaturedDataverse.equals(null));
+    }
+
+    @Test
+    public void testEqualsWithDifferentClass() {
+        DatasetFieldType datasetFieldType = new DatasetFieldType();
+
+        assertFalse(this.dataverseFeaturedDataverse.equals(datasetFieldType));
+    }
+
+    @Test
+    public void testEqualsWithSameClassSameId() {
+        DataverseFeaturedDataverse dataverseFeaturedDataverse1 = new DataverseFeaturedDataverse();
+        dataverseFeaturedDataverse1.setId(1L);
+
+        assertTrue(this.dataverseFeaturedDataverse.equals(dataverseFeaturedDataverse1));
+    }
+
+    @Test
+    public void testEqualsWithSameClassDifferentId() {
+        DataverseFeaturedDataverse dataverseFeaturedDataverse1 = new DataverseFeaturedDataverse();
+        dataverseFeaturedDataverse1.setId(2L);
+
+        assertFalse(this.dataverseFeaturedDataverse.equals(dataverseFeaturedDataverse1));
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/DataverseThemeTest.java b/src/test/java/edu/harvard/iq/dataverse/DataverseThemeTest.java
new file mode 100644
index 00000000000..e6721e34c6f
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/DataverseThemeTest.java
@@ -0,0 +1,47 @@
+package edu.harvard.iq.dataverse;
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+
+import org.junit.jupiter.api.BeforeEach;
+
+public class DataverseThemeTest {
+    private DataverseTheme dataverseTheme;
+
+    @BeforeEach
+    public void before() {
+        this.dataverseTheme = new DataverseTheme();
+        this.dataverseTheme.setId(1L);
+    }
+
+    @Test
+    public void testEqualsWithNull() {
+        assertFalse(this.dataverseTheme.equals(null));
+    }
+
+    @Test
+    public void testEqualsWithDifferentClass() {
+        DatasetFieldType datasetFieldType = new DatasetFieldType();
+
+        assertFalse(this.dataverseTheme.equals(datasetFieldType));
+    }
+
+    @Test
+    public void testEqualsWithSameClassSameId() {
+        DataverseTheme dataverseTheme1 = new DataverseTheme();
+        dataverseTheme1.setId(1L);
+
+        assertTrue(this.dataverseTheme.equals(dataverseTheme1));
+    }
+
+    @Test
+    public void testEqualsWithSameClassDifferentId() {
+        DataverseTheme dataverseTheme1 = new DataverseTheme();
+        dataverseTheme1.setId(2L);
+
+        assertFalse(this.dataverseTheme.equals(dataverseTheme1));
+    }
+}
\ No newline at end of file

From 56994b16b9a8dd92d550d491b5df00b542450e64 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Fri, 6 Oct 2023 00:55:09 +0200
Subject: [PATCH 527/922] Use JsonUtil.getJsonObject to prevent resource leaks

---
 .../harvard/iq/dataverse/api/Datasets.java    | 28 ++++++++-----------
 .../harvard/iq/dataverse/api/Dataverses.java  | 11 ++++----
 .../iq/dataverse/api/HarvestingClients.java   |  9 +++---
 .../edu/harvard/iq/dataverse/api/Prov.java    |  6 ++--
 .../api/imports/ImportServiceBean.java        | 15 ++++------
 .../dataverse/globus/GlobusServiceBean.java   | 21 ++++++++------
 .../DatasetMetricsServiceBean.java            |  9 ++----
 7 files changed, 45 insertions(+), 54 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 704ec8f1989..aacfb78cc2e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -93,8 +93,8 @@
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
 import edu.harvard.iq.dataverse.util.json.JsonLDTerm;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
-import edu.harvard.iq.dataverse.util.SignpostingResources;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import edu.harvard.iq.dataverse.util.SignpostingResources;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*;
@@ -109,7 +109,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.StringReader;
 import java.net.URI;
 import java.sql.Timestamp;
 import java.text.MessageFormat;
@@ -675,10 +674,10 @@ public Response updateDraftVersion(@Context ContainerRequestContext crc, String
             return error( Response.Status.BAD_REQUEST, "Only the :draft version can be updated");
         }
         
-        try ( StringReader rdr = new StringReader(jsonBody) ) {
+        try {
             DataverseRequest req = createDataverseRequest(getRequestUser(crc));
             Dataset ds = findDatasetOrDie(id);
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             DatasetVersion incomingVersion = jsonParser().parseDatasetVersion(json);
             
             // clear possibly stale fields from the incoming dataset version.
@@ -834,10 +833,10 @@ public Response deleteVersionMetadata(@Context ContainerRequestContext crc, Stri
     }
 
     private Response processDatasetFieldDataDelete(String jsonBody, String id, DataverseRequest req) {
-        try (StringReader rdr = new StringReader(jsonBody)) {
+        try {
 
             Dataset ds = findDatasetOrDie(id);
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             //Get the current draft or create a new version to update
             DatasetVersion dsv = ds.getOrCreateEditVersion();
             dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv);
@@ -991,10 +990,10 @@ public Response editVersionMetadata(@Context ContainerRequestContext crc, String
     
     
     private Response processDatasetUpdate(String jsonBody, String id, DataverseRequest req, Boolean replaceData){
-        try (StringReader rdr = new StringReader(jsonBody)) {
+        try {
            
             Dataset ds = findDatasetOrDie(id);
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             //Get the current draft or create a new version to update
             DatasetVersion dsv = ds.getOrCreateEditVersion();
             dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv);
@@ -1441,8 +1440,7 @@ public Response createFileEmbargo(@Context ContainerRequestContext crc, @PathPar
             return error(Status.BAD_REQUEST, "No Embargoes allowed");
         }
 
-        StringReader rdr = new StringReader(jsonBody);
-        JsonObject json = Json.createReader(rdr).readObject();
+        JsonObject json = JsonUtil.getJsonObject(jsonBody);
 
         Embargo embargo = new Embargo();
 
@@ -1585,8 +1583,7 @@ public Response removeFileEmbargo(@Context ContainerRequestContext crc, @PathPar
             return error(Status.BAD_REQUEST, "No Embargoes allowed");
         }
 
-        StringReader rdr = new StringReader(jsonBody);
-        JsonObject json = Json.createReader(rdr).readObject();
+        JsonObject json = JsonUtil.getJsonObject(jsonBody);
 
         List<DataFile> datasetFiles = dataset.getFiles();
         List<DataFile> embargoFilesToUnset = new LinkedList<>();
@@ -2101,8 +2098,7 @@ public Response returnToAuthor(@Context ContainerRequestContext crc, @PathParam(
         if (jsonBody == null || jsonBody.isEmpty()) {
             return error(Response.Status.BAD_REQUEST, "You must supply JSON to this API endpoint and it must contain a reason for returning the dataset (field: reasonForReturn).");
         }
-        StringReader rdr = new StringReader(jsonBody);
-        JsonObject json = Json.createReader(rdr).readObject();
+        JsonObject json = JsonUtil.getJsonObject(jsonBody);
         try {
             Dataset dataset = findDatasetOrDie(idSupplied);
             String reasonForReturn = null;
@@ -2354,9 +2350,7 @@ public Response completeMPUpload(@Context ContainerRequestContext crc, String pa
             List<PartETag> eTagList = new ArrayList<PartETag>();
             logger.info("Etags: " + partETagBody);
             try {
-                JsonReader jsonReader = Json.createReader(new StringReader(partETagBody));
-                JsonObject object = jsonReader.readObject();
-                jsonReader.close();
+                JsonObject object = JsonUtil.getJsonObject(partETagBody);
                 for (String partNo : object.keySet()) {
                     eTagList.add(new PartETag(Integer.parseInt(partNo), object.getString(partNo)));
                 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index a60775cbd38..c64ba42999c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -78,8 +78,9 @@
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import edu.harvard.iq.dataverse.util.json.JsonPrinter;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief;
-import java.io.StringReader;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -178,8 +179,8 @@ public Response addDataverse(@Context ContainerRequestContext crc, String body,
 
         Dataverse d;
         JsonObject dvJson;
-        try (StringReader rdr = new StringReader(body)) {
-            dvJson = Json.createReader(rdr).readObject();
+        try {
+            dvJson = JsonUtil.getJsonObject(body);
             d = jsonParser().parseDataverse(dvJson);
         } catch (JsonParsingException jpe) {
             logger.log(Level.SEVERE, "Json: {0}", body);
@@ -559,8 +560,8 @@ public Response recreateDataset(@Context ContainerRequestContext crc, String jso
     }
     
     private Dataset parseDataset(String datasetJson) throws WrappedResponse {
-        try (StringReader rdr = new StringReader(datasetJson)) {
-            return jsonParser().parseDataset(Json.createReader(rdr).readObject());
+        try {
+            return jsonParser().parseDataset(JsonUtil.getJsonObject(datasetJson));
         } catch (JsonParsingException | JsonParseException jpe) {
             logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", datasetJson);
             throw new WrappedResponse(error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()));
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
index d7eec9f5757..dfc9f48dd1a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
@@ -16,6 +16,7 @@
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import edu.harvard.iq.dataverse.util.json.JsonPrinter;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import jakarta.json.JsonObjectBuilder;
 import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
 import java.io.IOException;
@@ -164,8 +165,8 @@ public Response createHarvestingClient(@Context ContainerRequestContext crc, Str
             return wr.getResponse();
         }
  
-        try ( StringReader rdr = new StringReader(jsonBody) ) {
-            JsonObject json = Json.createReader(rdr).readObject();
+        try {
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             
             // Check that the client with this name doesn't exist yet: 
             // (we could simply let the command fail, but that does not result 
@@ -261,9 +262,9 @@ public Response modifyHarvestingClient(@Context ContainerRequestContext crc, Str
         
         String ownerDataverseAlias = harvestingClient.getDataverse().getAlias();
         
-        try ( StringReader rdr = new StringReader(jsonBody) ) {
+        try {
             DataverseRequest req = createDataverseRequest(getRequestUser(crc));
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             
             HarvestingClient newHarvestingClient = new HarvestingClient(); 
             String newDataverseAlias = jsonParser().parseHarvestingClient(json, newHarvestingClient);
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Prov.java b/src/main/java/edu/harvard/iq/dataverse/api/Prov.java
index 37b4792920f..7f81ca20988 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Prov.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Prov.java
@@ -12,7 +12,8 @@
 import edu.harvard.iq.dataverse.engine.command.impl.PersistProvJsonCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
 import edu.harvard.iq.dataverse.util.BundleUtil;
-import java.io.StringReader;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import java.util.HashMap;
 import java.util.logging.Logger;
 import jakarta.inject.Inject;
@@ -109,11 +110,10 @@ public Response addProvFreeForm(@Context ContainerRequestContext crc, String bod
         if(!systemConfig.isProvCollectionEnabled()) {
             return error(FORBIDDEN, BundleUtil.getStringFromBundle("api.prov.error.provDisabled"));
         }
-        StringReader rdr = new StringReader(body);
         JsonObject jsonObj = null;
         
         try {
-            jsonObj = Json.createReader(rdr).readObject();
+            jsonObj = JsonUtil.getJsonObject(body);
         } catch (JsonException ex) {
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("api.prov.error.freeformInvalidJson"));
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index bcb67b180c8..c17ba909230 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -36,12 +36,12 @@
 import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import edu.harvard.iq.dataverse.util.json.JsonParser;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.io.StringReader;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Date;
@@ -60,7 +60,6 @@
 import jakarta.json.Json;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
-import jakarta.json.JsonReader;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
 import jakarta.validation.ConstraintViolation;
@@ -259,9 +258,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 throw new ImportException("Failed to transform XML metadata format "+metadataFormat+" into a DatasetDTO");
             }
         }
-        
-        JsonReader jsonReader = Json.createReader(new StringReader(json));
-        JsonObject obj = jsonReader.readObject();
+
+        JsonObject obj = JsonUtil.getJsonObject(json);
         //and call parse Json to read it into a dataset   
         try {
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, harvestingClient);
@@ -396,10 +394,8 @@ public JsonObject ddiToJson(String xmlToParse) throws ImportException, XMLStream
         // convert DTO to Json,
         Gson gson = new GsonBuilder().setPrettyPrinting().create();
         String json = gson.toJson(dsDTO);
-        JsonReader jsonReader = Json.createReader(new StringReader(json));
-        JsonObject obj = jsonReader.readObject();
 
-        return obj;
+        return JsonUtil.getJsonObject(json);
     }
     
     public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse owner, String xmlToParse, String fileName, ImportType importType, PrintWriter cleanupLog) throws ImportException, IOException {
@@ -416,8 +412,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o
         // convert DTO to Json, 
         Gson gson = new GsonBuilder().setPrettyPrinting().create();
         String json = gson.toJson(dsDTO);
-        JsonReader jsonReader = Json.createReader(new StringReader(json));
-        JsonObject obj = jsonReader.readObject();
+        JsonObject obj = JsonUtil.getJsonObject(json);
         //and call parse Json to read it into a dataset   
         try {
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService);
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index d2613422be9..d6943ec3511 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -596,13 +596,14 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
         Thread.sleep(5000);
 
         JsonObject jsonObject = null;
-        try (StringReader rdr = new StringReader(jsonData)) {
-            jsonObject = Json.createReader(rdr).readObject();
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
         } catch (Exception jpe) {
             jpe.printStackTrace();
-            logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}");
+            logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonData);
+            // TODO: I think an (parsing) exception should stop the process, shouldn't it?
         }
-        logger.info("json: " + JsonUtil.prettyPrint(jsonObject));
+        logger.log(Level.INFO, "json: {0}", JsonUtil.prettyPrint(jsonObject));
 
         String taskIdentifier = jsonObject.getString("taskIdentifier");
 
@@ -808,11 +809,12 @@ private String addFiles(String curlCommand, Logger globusLogger) {
                 sb.append(line);
             globusLogger.info(" API Output :  " + sb.toString());
             JsonObject jsonObject = null;
-            try (StringReader rdr = new StringReader(sb.toString())) {
-                jsonObject = Json.createReader(rdr).readObject();
+            try {
+                jsonObject = JsonUtil.getJsonObject(sb.toString());
             } catch (Exception jpe) {
                 jpe.printStackTrace();
                 globusLogger.log(Level.SEVERE, "Error parsing dataset json.");
+                // TODO: a parsing exception should cause the process to stop.
             }
 
             status = jsonObject.getString("status");
@@ -853,11 +855,12 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         globusLogger.info("Starting an globusDownload ");
 
         JsonObject jsonObject = null;
-        try (StringReader rdr = new StringReader(jsonData)) {
-            jsonObject = Json.createReader(rdr).readObject();
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
         } catch (Exception jpe) {
             jpe.printStackTrace();
-            globusLogger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}");
+            globusLogger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonData);
+            // TODO: stop the process after this parsing exception.
         }
 
         String taskIdentifier = jsonObject.getString("taskIdentifier");
diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java
index 0925c164bf4..0fb7e9f1e6c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java
@@ -3,7 +3,8 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
-import java.io.StringReader;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
@@ -14,10 +15,8 @@
 import jakarta.ejb.EJBException;
 import jakarta.ejb.Stateless;
 import jakarta.inject.Named;
-import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
-import jakarta.json.JsonReader;
 import jakarta.json.JsonValue;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
@@ -125,9 +124,7 @@ public List<DatasetMetrics> parseSushiReport(JsonObject report, Dataset dataset)
             List<DatasetMetrics> datasetMetricsDataset = new ArrayList<>();
             String globalId = null;
             Dataset ds = null;
-            StringReader rdr = new StringReader(reportDataset.toString());
-            JsonReader jrdr = Json.createReader(rdr);
-            JsonObject obj = jrdr.readObject();
+            JsonObject obj = JsonUtil.getJsonObject(reportDataset.toString());
             String jsonGlobalId = "";
             String globalIdType = "";
             if (obj.containsKey("dataset-id")) {

From 7d72db1778802a72022cd625ddd545290d96405b Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Fri, 6 Oct 2023 00:55:09 +0200
Subject: [PATCH 528/922] Add

---
 .../harvard/iq/dataverse/api/Datasets.java    | 28 ++++++++-----------
 .../harvard/iq/dataverse/api/Dataverses.java  | 11 ++++----
 .../iq/dataverse/api/HarvestingClients.java   |  9 +++---
 .../edu/harvard/iq/dataverse/api/Prov.java    |  6 ++--
 .../api/imports/ImportServiceBean.java        | 15 ++++------
 .../dataverse/globus/GlobusServiceBean.java   | 21 ++++++++------
 .../DatasetMetricsServiceBean.java            |  9 ++----
 .../settings/SettingsServiceBean.java         | 16 ++++-------
 .../iq/dataverse/workflows/WorkflowUtil.java  |  7 ++---
 9 files changed, 54 insertions(+), 68 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 704ec8f1989..aacfb78cc2e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -93,8 +93,8 @@
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
 import edu.harvard.iq.dataverse.util.json.JsonLDTerm;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
-import edu.harvard.iq.dataverse.util.SignpostingResources;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import edu.harvard.iq.dataverse.util.SignpostingResources;
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.*;
@@ -109,7 +109,6 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.StringReader;
 import java.net.URI;
 import java.sql.Timestamp;
 import java.text.MessageFormat;
@@ -675,10 +674,10 @@ public Response updateDraftVersion(@Context ContainerRequestContext crc, String
             return error( Response.Status.BAD_REQUEST, "Only the :draft version can be updated");
         }
         
-        try ( StringReader rdr = new StringReader(jsonBody) ) {
+        try {
             DataverseRequest req = createDataverseRequest(getRequestUser(crc));
             Dataset ds = findDatasetOrDie(id);
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             DatasetVersion incomingVersion = jsonParser().parseDatasetVersion(json);
             
             // clear possibly stale fields from the incoming dataset version.
@@ -834,10 +833,10 @@ public Response deleteVersionMetadata(@Context ContainerRequestContext crc, Stri
     }
 
     private Response processDatasetFieldDataDelete(String jsonBody, String id, DataverseRequest req) {
-        try (StringReader rdr = new StringReader(jsonBody)) {
+        try {
 
             Dataset ds = findDatasetOrDie(id);
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             //Get the current draft or create a new version to update
             DatasetVersion dsv = ds.getOrCreateEditVersion();
             dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv);
@@ -991,10 +990,10 @@ public Response editVersionMetadata(@Context ContainerRequestContext crc, String
     
     
     private Response processDatasetUpdate(String jsonBody, String id, DataverseRequest req, Boolean replaceData){
-        try (StringReader rdr = new StringReader(jsonBody)) {
+        try {
            
             Dataset ds = findDatasetOrDie(id);
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             //Get the current draft or create a new version to update
             DatasetVersion dsv = ds.getOrCreateEditVersion();
             dsv.getTermsOfUseAndAccess().setDatasetVersion(dsv);
@@ -1441,8 +1440,7 @@ public Response createFileEmbargo(@Context ContainerRequestContext crc, @PathPar
             return error(Status.BAD_REQUEST, "No Embargoes allowed");
         }
 
-        StringReader rdr = new StringReader(jsonBody);
-        JsonObject json = Json.createReader(rdr).readObject();
+        JsonObject json = JsonUtil.getJsonObject(jsonBody);
 
         Embargo embargo = new Embargo();
 
@@ -1585,8 +1583,7 @@ public Response removeFileEmbargo(@Context ContainerRequestContext crc, @PathPar
             return error(Status.BAD_REQUEST, "No Embargoes allowed");
         }
 
-        StringReader rdr = new StringReader(jsonBody);
-        JsonObject json = Json.createReader(rdr).readObject();
+        JsonObject json = JsonUtil.getJsonObject(jsonBody);
 
         List<DataFile> datasetFiles = dataset.getFiles();
         List<DataFile> embargoFilesToUnset = new LinkedList<>();
@@ -2101,8 +2098,7 @@ public Response returnToAuthor(@Context ContainerRequestContext crc, @PathParam(
         if (jsonBody == null || jsonBody.isEmpty()) {
             return error(Response.Status.BAD_REQUEST, "You must supply JSON to this API endpoint and it must contain a reason for returning the dataset (field: reasonForReturn).");
         }
-        StringReader rdr = new StringReader(jsonBody);
-        JsonObject json = Json.createReader(rdr).readObject();
+        JsonObject json = JsonUtil.getJsonObject(jsonBody);
         try {
             Dataset dataset = findDatasetOrDie(idSupplied);
             String reasonForReturn = null;
@@ -2354,9 +2350,7 @@ public Response completeMPUpload(@Context ContainerRequestContext crc, String pa
             List<PartETag> eTagList = new ArrayList<PartETag>();
             logger.info("Etags: " + partETagBody);
             try {
-                JsonReader jsonReader = Json.createReader(new StringReader(partETagBody));
-                JsonObject object = jsonReader.readObject();
-                jsonReader.close();
+                JsonObject object = JsonUtil.getJsonObject(partETagBody);
                 for (String partNo : object.keySet()) {
                     eTagList.add(new PartETag(Integer.parseInt(partNo), object.getString(partNo)));
                 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index a60775cbd38..c64ba42999c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -78,8 +78,9 @@
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import edu.harvard.iq.dataverse.util.json.JsonPrinter;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief;
-import java.io.StringReader;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -178,8 +179,8 @@ public Response addDataverse(@Context ContainerRequestContext crc, String body,
 
         Dataverse d;
         JsonObject dvJson;
-        try (StringReader rdr = new StringReader(body)) {
-            dvJson = Json.createReader(rdr).readObject();
+        try {
+            dvJson = JsonUtil.getJsonObject(body);
             d = jsonParser().parseDataverse(dvJson);
         } catch (JsonParsingException jpe) {
             logger.log(Level.SEVERE, "Json: {0}", body);
@@ -559,8 +560,8 @@ public Response recreateDataset(@Context ContainerRequestContext crc, String jso
     }
     
     private Dataset parseDataset(String datasetJson) throws WrappedResponse {
-        try (StringReader rdr = new StringReader(datasetJson)) {
-            return jsonParser().parseDataset(Json.createReader(rdr).readObject());
+        try {
+            return jsonParser().parseDataset(JsonUtil.getJsonObject(datasetJson));
         } catch (JsonParsingException | JsonParseException jpe) {
             logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", datasetJson);
             throw new WrappedResponse(error(Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()));
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
index d7eec9f5757..dfc9f48dd1a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/HarvestingClients.java
@@ -16,6 +16,7 @@
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import edu.harvard.iq.dataverse.util.json.JsonPrinter;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import jakarta.json.JsonObjectBuilder;
 import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder;
 import java.io.IOException;
@@ -164,8 +165,8 @@ public Response createHarvestingClient(@Context ContainerRequestContext crc, Str
             return wr.getResponse();
         }
  
-        try ( StringReader rdr = new StringReader(jsonBody) ) {
-            JsonObject json = Json.createReader(rdr).readObject();
+        try {
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             
             // Check that the client with this name doesn't exist yet: 
             // (we could simply let the command fail, but that does not result 
@@ -261,9 +262,9 @@ public Response modifyHarvestingClient(@Context ContainerRequestContext crc, Str
         
         String ownerDataverseAlias = harvestingClient.getDataverse().getAlias();
         
-        try ( StringReader rdr = new StringReader(jsonBody) ) {
+        try {
             DataverseRequest req = createDataverseRequest(getRequestUser(crc));
-            JsonObject json = Json.createReader(rdr).readObject();
+            JsonObject json = JsonUtil.getJsonObject(jsonBody);
             
             HarvestingClient newHarvestingClient = new HarvestingClient(); 
             String newDataverseAlias = jsonParser().parseHarvestingClient(json, newHarvestingClient);
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Prov.java b/src/main/java/edu/harvard/iq/dataverse/api/Prov.java
index 37b4792920f..7f81ca20988 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Prov.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Prov.java
@@ -12,7 +12,8 @@
 import edu.harvard.iq.dataverse.engine.command.impl.PersistProvJsonCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
 import edu.harvard.iq.dataverse.util.BundleUtil;
-import java.io.StringReader;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import java.util.HashMap;
 import java.util.logging.Logger;
 import jakarta.inject.Inject;
@@ -109,11 +110,10 @@ public Response addProvFreeForm(@Context ContainerRequestContext crc, String bod
         if(!systemConfig.isProvCollectionEnabled()) {
             return error(FORBIDDEN, BundleUtil.getStringFromBundle("api.prov.error.provDisabled"));
         }
-        StringReader rdr = new StringReader(body);
         JsonObject jsonObj = null;
         
         try {
-            jsonObj = Json.createReader(rdr).readObject();
+            jsonObj = JsonUtil.getJsonObject(body);
         } catch (JsonException ex) {
             return error(BAD_REQUEST, BundleUtil.getStringFromBundle("api.prov.error.freeformInvalidJson"));
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index bcb67b180c8..c17ba909230 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -36,12 +36,12 @@
 import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
 import edu.harvard.iq.dataverse.util.json.JsonParseException;
 import edu.harvard.iq.dataverse.util.json.JsonParser;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.license.LicenseServiceBean;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.io.StringReader;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Date;
@@ -60,7 +60,6 @@
 import jakarta.json.Json;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
-import jakarta.json.JsonReader;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
 import jakarta.validation.ConstraintViolation;
@@ -259,9 +258,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
                 throw new ImportException("Failed to transform XML metadata format "+metadataFormat+" into a DatasetDTO");
             }
         }
-        
-        JsonReader jsonReader = Json.createReader(new StringReader(json));
-        JsonObject obj = jsonReader.readObject();
+
+        JsonObject obj = JsonUtil.getJsonObject(json);
         //and call parse Json to read it into a dataset   
         try {
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, harvestingClient);
@@ -396,10 +394,8 @@ public JsonObject ddiToJson(String xmlToParse) throws ImportException, XMLStream
         // convert DTO to Json,
         Gson gson = new GsonBuilder().setPrettyPrinting().create();
         String json = gson.toJson(dsDTO);
-        JsonReader jsonReader = Json.createReader(new StringReader(json));
-        JsonObject obj = jsonReader.readObject();
 
-        return obj;
+        return JsonUtil.getJsonObject(json);
     }
     
     public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse owner, String xmlToParse, String fileName, ImportType importType, PrintWriter cleanupLog) throws ImportException, IOException {
@@ -416,8 +412,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o
         // convert DTO to Json, 
         Gson gson = new GsonBuilder().setPrettyPrinting().create();
         String json = gson.toJson(dsDTO);
-        JsonReader jsonReader = Json.createReader(new StringReader(json));
-        JsonObject obj = jsonReader.readObject();
+        JsonObject obj = JsonUtil.getJsonObject(json);
         //and call parse Json to read it into a dataset   
         try {
             JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService);
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index d2613422be9..d6943ec3511 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -596,13 +596,14 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
         Thread.sleep(5000);
 
         JsonObject jsonObject = null;
-        try (StringReader rdr = new StringReader(jsonData)) {
-            jsonObject = Json.createReader(rdr).readObject();
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
         } catch (Exception jpe) {
             jpe.printStackTrace();
-            logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}");
+            logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonData);
+            // TODO: I think an (parsing) exception should stop the process, shouldn't it?
         }
-        logger.info("json: " + JsonUtil.prettyPrint(jsonObject));
+        logger.log(Level.INFO, "json: {0}", JsonUtil.prettyPrint(jsonObject));
 
         String taskIdentifier = jsonObject.getString("taskIdentifier");
 
@@ -808,11 +809,12 @@ private String addFiles(String curlCommand, Logger globusLogger) {
                 sb.append(line);
             globusLogger.info(" API Output :  " + sb.toString());
             JsonObject jsonObject = null;
-            try (StringReader rdr = new StringReader(sb.toString())) {
-                jsonObject = Json.createReader(rdr).readObject();
+            try {
+                jsonObject = JsonUtil.getJsonObject(sb.toString());
             } catch (Exception jpe) {
                 jpe.printStackTrace();
                 globusLogger.log(Level.SEVERE, "Error parsing dataset json.");
+                // TODO: a parsing exception should cause the process to stop.
             }
 
             status = jsonObject.getString("status");
@@ -853,11 +855,12 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         globusLogger.info("Starting an globusDownload ");
 
         JsonObject jsonObject = null;
-        try (StringReader rdr = new StringReader(jsonData)) {
-            jsonObject = Json.createReader(rdr).readObject();
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
         } catch (Exception jpe) {
             jpe.printStackTrace();
-            globusLogger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}");
+            globusLogger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}", jsonData);
+            // TODO: stop the process after this parsing exception.
         }
 
         String taskIdentifier = jsonObject.getString("taskIdentifier");
diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java
index 0925c164bf4..0fb7e9f1e6c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetMetricsServiceBean.java
@@ -3,7 +3,8 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetServiceBean;
-import java.io.StringReader;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
@@ -14,10 +15,8 @@
 import jakarta.ejb.EJBException;
 import jakarta.ejb.Stateless;
 import jakarta.inject.Named;
-import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
-import jakarta.json.JsonReader;
 import jakarta.json.JsonValue;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
@@ -125,9 +124,7 @@ public List<DatasetMetrics> parseSushiReport(JsonObject report, Dataset dataset)
             List<DatasetMetrics> datasetMetricsDataset = new ArrayList<>();
             String globalId = null;
             Dataset ds = null;
-            StringReader rdr = new StringReader(reportDataset.toString());
-            JsonReader jrdr = Json.createReader(rdr);
-            JsonObject obj = jrdr.readObject();
+            JsonObject obj = JsonUtil.getJsonObject(reportDataset.toString());
             String jsonGlobalId = "";
             String globalIdType = "";
             if (obj.containsKey("dataset-id")) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
index 0aa403a5116..6b74810eb53 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
@@ -4,14 +4,12 @@
 import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean;
 import edu.harvard.iq.dataverse.api.ApiBlockingFilter;
 import edu.harvard.iq.dataverse.util.StringUtil;
-
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import jakarta.ejb.EJB;
 import jakarta.ejb.Stateless;
 import jakarta.inject.Named;
-import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
-import jakarta.json.JsonReader;
 import jakarta.json.JsonValue;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
@@ -20,7 +18,6 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import java.io.StringReader;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -696,8 +693,8 @@ public Long getValueForCompoundKeyAsLong(Key key, String param){
     	   try {
     		   return Long.parseLong(val);
     	   } catch (NumberFormatException ex) {
-    		   try ( StringReader rdr = new StringReader(val) ) {
-    			   JsonObject settings = Json.createReader(rdr).readObject();
+    		   try {
+    			   JsonObject settings = JsonUtil.getJsonObject(val);
     			   if(settings.containsKey(param)) {
     				   return Long.parseLong(settings.getString(param));
     			   } else if(settings.containsKey("default")) {
@@ -730,8 +727,8 @@ public Boolean getValueForCompoundKeyAsBoolean(Key key, String param) {
             return null;
         }
 
-        try (StringReader rdr = new StringReader(val)) {
-            JsonObject settings = Json.createReader(rdr).readObject();
+        try {
+            JsonObject settings = JsonUtil.getJsonObject(val);
             if (settings.containsKey(param)) {
                 return Boolean.parseBoolean(settings.getString(param));
             } else if (settings.containsKey("default")) {
@@ -897,8 +894,7 @@ public Map<String, String> getBaseMetadataLanguageMap(Map<String,String> languag
             if(mlString.isEmpty()) {
                 mlString="[]";
             }
-            JsonReader jsonReader = Json.createReader(new StringReader(mlString));
-            JsonArray languages = jsonReader.readArray();
+            JsonArray languages = JsonUtil.getJsonArray(mlString);
             for(JsonValue jv: languages) {
                 JsonObject lang = (JsonObject) jv;
                 languageMap.put(lang.getString("locale"), lang.getString("title"));
diff --git a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java
index 456b829ba61..b104f113db2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/workflows/WorkflowUtil.java
@@ -3,7 +3,6 @@
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.api.Util;
 
-import java.io.StringReader;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -11,7 +10,7 @@
 import jakarta.json.Json;
 import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObject;
-
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import edu.harvard.iq.dataverse.workflow.step.Failure;
 import edu.harvard.iq.dataverse.workflow.step.Success;
@@ -42,8 +41,8 @@ public static JsonArrayBuilder getAllWorkflowComments(DatasetVersion datasetVers
     }
 
     public static WorkflowStepResult parseResponse(String externalData) {
-        try (StringReader reader = new StringReader(externalData)) {
-            JsonObject response = Json.createReader(reader).readObject();
+        try {
+            JsonObject response = JsonUtil.getJsonObject(externalData);
             String status = null;
             //Lower case is documented, upper case is deprecated
             if(response.containsKey("status")) {

From faa8de1639b007a8c2a9f90b73a723fc10e7f19a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 6 Oct 2023 10:31:28 +0100
Subject: [PATCH 529/922] Changed: using json files in curl examples for
 tabular tags and categories update in the docs

---
 doc/sphinx-guides/source/api/native-api.rst | 54 +++++++++++++++------
 1 file changed, 38 insertions(+), 16 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index f78ae62f3ae..97b41ffa98a 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3026,6 +3026,15 @@ Updates the categories for an existing file where ``ID`` is the database id of t
 
 Although updating categories can also be done with the previous endpoint, this has been created to be more practical when it is only necessary to update categories and not other metadata fields.
 
+The JSON representation of file categories (``categories.json``) looks like this::
+
+  {
+    "categories": [
+      "Data",
+      "Custom"
+    ]
+  }
+
 A curl example using an ``ID``
 
 .. code-block:: bash
@@ -3033,18 +3042,19 @@ A curl example using an ``ID``
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
+  export FILE_PATH=categories.json
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "$SERVER_URL/api/files/$ID/metadata/categories"
+    "$SERVER_URL/api/files/$ID/metadata/categories" \
+    -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "http://demo.dataverse.org/api/files/24/metadata/categories"
+    "http://demo.dataverse.org/api/files/24/metadata/categories" \
+    -H "Content-type:application/json" --upload-file categories.json
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -3053,18 +3063,19 @@ A curl example using a ``PERSISTENT_ID``
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+  export FILE_PATH=categories.json
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID"
+    "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID" \
+    -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"categories":["Category1","Category2"]}' \
-    "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000"
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000" \
+    -H "Content-type:application/json" --upload-file categories.json
 
 Note that if the specified categories do not exist, they will be created.
 
@@ -3073,6 +3084,15 @@ Updating File Tabular Tags
 
 Updates the tabular tags for an existing tabular file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the tabular tag names.
 
+The JSON representation of tabular tags (``tags.json``) looks like this::
+
+  {
+    "tabularTags": [
+      "Survey",
+      "Genomics"
+    ]
+  }
+
 A curl example using an ``ID``
 
 .. code-block:: bash
@@ -3080,18 +3100,19 @@ A curl example using an ``ID``
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
+  export FILE_PATH=tags.json
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
-    "$SERVER_URL/api/files/$ID/metadata/tabularTags"
+    "$SERVER_URL/api/files/$ID/metadata/tabularTags" \
+    -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
-    "http://demo.dataverse.org/api/files/24/metadata/tabularTags"
+    "http://demo.dataverse.org/api/files/24/metadata/tabularTags" \
+    -H "Content-type:application/json" --upload-file tags.json
 
 A curl example using a ``PERSISTENT_ID``
 
@@ -3100,18 +3121,19 @@ A curl example using a ``PERSISTENT_ID``
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+  export FILE_PATH=tags.json
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
-    "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID"
+    "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID" \
+    -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"tabularTags":["Survey","Genomics"]}' \
-    "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000"
+    "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000" \
+    -H "Content-type:application/json" --upload-file tags.json
 
 Note that the specified tabular tags must be valid. The supported tags are:
 

From 1440e653b8480c754f0669bb15f1b2cd92442522 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 6 Oct 2023 14:48:30 +0100
Subject: [PATCH 530/922] Refactor: FileSearchCriteria to encapsulate all
 criteria options

---
 .../DatasetVersionFilesServiceBean.java       | 54 +++++++++----------
 .../iq/dataverse/FileSearchCriteria.java      | 45 ++++++++++++++++
 .../harvard/iq/dataverse/api/Datasets.java    | 17 ++++--
 .../iq/dataverse/util/json/JsonPrinter.java   |  5 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 24 +++++----
 5 files changed, 97 insertions(+), 48 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 6006d937100..a436b10d340 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -29,6 +29,8 @@
 
 import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
 
+import edu.harvard.iq.dataverse.FileSearchCriteria.FileAccessStatus;
+
 @Stateless
 @Named
 public class DatasetVersionFilesServiceBean implements Serializable {
@@ -44,17 +46,10 @@ public class DatasetVersionFilesServiceBean implements Serializable {
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
      */
-    public enum FileMetadatasOrderCriteria {
+    public enum FileOrderCriteria {
         NameAZ, NameZA, Newest, Oldest, Size, Type
     }
 
-    /**
-     * Status of the particular DataFile based on active embargoes and restriction state used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
-     */
-    public enum DataFileAccessStatus {
-        Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
-    }
-
     /**
      * Given a DatasetVersion, returns its total file metadata count
      *
@@ -107,17 +102,17 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
     }
 
     /**
-     * Given a DatasetVersion, returns its file metadata count per DataFileAccessStatus
+     * Given a DatasetVersion, returns its file metadata count per FileAccessStatus
      *
      * @param datasetVersion the DatasetVersion to access
-     * @return Map<DataFileAccessStatus, Long> of file metadata counts per DataFileAccessStatus
+     * @return Map<FileAccessStatus, Long> of file metadata counts per FileAccessStatus
      */
-    public Map<DataFileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) {
-        Map<DataFileAccessStatus, Long> allCounts = new HashMap<>();
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Public);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Restricted);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenPublic);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenRestricted);
+    public Map<FileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) {
+        Map<FileAccessStatus, Long> allCounts = new HashMap<>();
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.Public);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.Restricted);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.EmbargoedThenPublic);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.EmbargoedThenRestricted);
         return allCounts;
     }
 
@@ -127,29 +122,30 @@ public Map<DataFileAccessStatus, Long> getFileMetadataCountPerAccessStatus(Datas
      * @param datasetVersion the DatasetVersion to access
      * @param limit          for pagination, can be null
      * @param offset         for pagination, can be null
-     * @param contentType    for retrieving only files with this content type
-     * @param accessStatus   for retrieving only files with this DataFileAccessStatus
-     * @param categoryName   for retrieving only files categorized with this category name
-     * @param tabularTagName for retrieving only files categorized with this tabular tag name
-     * @param searchText     for retrieving only files that contain the specified text within their labels or descriptions
-     * @param orderCriteria  a FileMetadatasOrderCriteria to order the results
+     * @param searchCriteria for retrieving only files matching this criteria
+     * @param orderCriteria  a FileOrderCriteria to order the results
      * @return a FileMetadata list from the specified DatasetVersion
      */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText, FileMetadatasOrderCriteria orderCriteria) {
+    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileSearchCriteria searchCriteria, FileOrderCriteria orderCriteria) {
         JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
 
+        String contentType = searchCriteria.getContentType();
         if (contentType != null) {
             baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
         }
+        FileAccessStatus accessStatus = searchCriteria.getAccessStatus();
         if (accessStatus != null) {
             baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
         }
+        String categoryName = searchCriteria.getCategoryName();
         if (categoryName != null) {
             baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
         }
+        String tabularTagName = searchCriteria.getTabularTagName();
         if (tabularTagName != null) {
             baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
         }
+        String searchText = searchCriteria.getSearchText();
         if (searchText != null && !searchText.isEmpty()) {
             searchText = searchText.trim().toLowerCase();
             baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
@@ -167,14 +163,14 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
         return baseQuery.fetch();
     }
 
-    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<DataFileAccessStatus, Long> totalCounts, DataFileAccessStatus dataFileAccessStatus) {
+    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<FileAccessStatus, Long> totalCounts, FileAccessStatus dataFileAccessStatus) {
         long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus);
         if (fileMetadataCount > 0) {
             totalCounts.put(dataFileAccessStatus, fileMetadataCount);
         }
     }
 
-    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, DataFileAccessStatus accessStatus) {
+    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, FileAccessStatus accessStatus) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
         return queryFactory
                 .selectFrom(fileMetadata)
@@ -182,16 +178,16 @@ private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, D
                 .stream().count();
     }
 
-    private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) {
+    private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileOrderCriteria orderCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
         JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
-        if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) {
+        if (orderCriteria == FileOrderCriteria.Newest || orderCriteria == FileOrderCriteria.Oldest) {
             baseQuery.from(dvObject).where(dvObject.id.eq(fileMetadata.dataFile.id));
         }
         return baseQuery;
     }
 
-    private BooleanExpression createGetFileMetadatasAccessStatusExpression(DataFileAccessStatus accessStatus) {
+    private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAccessStatus accessStatus) {
         QEmbargo embargo = fileMetadata.dataFile.embargo;
         BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
         BooleanExpression inactivelyEmbargoedExpression = embargo.isNull();
@@ -215,7 +211,7 @@ private BooleanExpression createGetFileMetadatasAccessStatusExpression(DataFileA
         return accessStatusExpression;
     }
 
-    private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileMetadatasOrderCriteria orderCriteria) {
+    private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileOrderCriteria orderCriteria) {
         DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
         switch (orderCriteria) {
             case NameZA:
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java b/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java
new file mode 100644
index 00000000000..62f10c18bdf
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java
@@ -0,0 +1,45 @@
+package edu.harvard.iq.dataverse;
+
+public class FileSearchCriteria {
+
+    private final String contentType;
+    private final FileAccessStatus accessStatus;
+    private final String categoryName;
+    private final String tabularTagName;
+    private final String searchText;
+
+    /**
+     * Status of the particular DataFile based on active embargoes and restriction state
+     */
+    public enum FileAccessStatus {
+        Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic
+    }
+
+    public FileSearchCriteria(String contentType, FileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText) {
+        this.contentType = contentType;
+        this.accessStatus = accessStatus;
+        this.categoryName = categoryName;
+        this.tabularTagName = tabularTagName;
+        this.searchText = searchText;
+    }
+
+    public String getContentType() {
+        return contentType;
+    }
+
+    public FileAccessStatus getAccessStatus() {
+        return accessStatus;
+    }
+
+    public String getCategoryName() {
+        return categoryName;
+    }
+
+    public String getTabularTagName() {
+        return tabularTagName;
+    }
+
+    public String getSearchText() {
+        return searchText;
+    }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b3be55399d8..14fd1b2453c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -506,19 +506,26 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                                     @Context HttpHeaders headers) {
         return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
-            DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria fileMetadatasOrderCriteria;
+            DatasetVersionFilesServiceBean.FileOrderCriteria fileOrderCriteria;
             try {
-                fileMetadatasOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameAZ;
+                fileOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ;
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            DatasetVersionFilesServiceBean.DataFileAccessStatus dataFileAccessStatus;
+            FileSearchCriteria.FileAccessStatus dataFileAccessStatus;
             try {
-                dataFileAccessStatus = accessStatus != null ? DatasetVersionFilesServiceBean.DataFileAccessStatus.valueOf(accessStatus) : null;
+                dataFileAccessStatus = accessStatus != null ? FileSearchCriteria.FileAccessStatus.valueOf(accessStatus) : null;
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
             }
-            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, tabularTagName, searchText, fileMetadatasOrderCriteria)));
+            FileSearchCriteria fileSearchCriteria = new FileSearchCriteria(
+                    contentType,
+                    dataFileAccessStatus,
+                    categoryName,
+                    tabularTagName,
+                    searchText
+            );
+            return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, fileSearchCriteria, fileOrderCriteria)));
         }, getRequestUser(crc));
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 1fed0b233e4..70840c7502f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -41,7 +41,6 @@
 import jakarta.json.Json;
 import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObjectBuilder;
-import jakarta.json.JsonValue;
 
 import java.util.function.BiConsumer;
 import java.util.function.BinaryOperator;
@@ -1108,9 +1107,9 @@ public static JsonObjectBuilder json(Map<String, Long> map) {
         return jsonObjectBuilder;
     }
 
-    public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> map) {
+    public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map<FileSearchCriteria.FileAccessStatus, Long> map) {
         JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-        for (Map.Entry<DatasetVersionFilesServiceBean.DataFileAccessStatus, Long> mapEntry : map.entrySet()) {
+        for (Map.Entry<FileSearchCriteria.FileAccessStatus, Long> mapEntry : map.entrySet()) {
             jsonObjectBuilder.add(mapEntry.getKey().toString(), mapEntry.getValue());
         }
         return jsonObjectBuilder;
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index b9f09cc7c07..5d1a89aa555 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -1,6 +1,7 @@
 package edu.harvard.iq.dataverse.api;
 
 import edu.harvard.iq.dataverse.DatasetVersionFilesServiceBean;
+import edu.harvard.iq.dataverse.FileSearchCriteria;
 import io.restassured.RestAssured;
 
 import static io.restassured.RestAssured.given;
@@ -3267,6 +3268,7 @@ public void getDatasetVersionCitation() {
                 .body("data.message", containsString("DRAFT VERSION"));
     }
 
+    @Test
     public void getVersionFiles() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
@@ -3334,7 +3336,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
         assertEquals(1, fileMetadatasCount);
 
         // Test NameZA order criteria
-        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.NameZA.toString(), apiToken);
+        Response getVersionFilesResponseNameZACriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileOrderCriteria.NameZA.toString(), apiToken);
 
         getVersionFilesResponseNameZACriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3345,7 +3347,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Newest order criteria
-        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Newest.toString(), apiToken);
+        Response getVersionFilesResponseNewestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileOrderCriteria.Newest.toString(), apiToken);
 
         getVersionFilesResponseNewestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3356,7 +3358,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName1));
 
         // Test Oldest order criteria
-        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Oldest.toString(), apiToken);
+        Response getVersionFilesResponseOldestCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileOrderCriteria.Oldest.toString(), apiToken);
 
         getVersionFilesResponseOldestCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3367,7 +3369,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Size order criteria
-        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Size.toString(), apiToken);
+        Response getVersionFilesResponseSizeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileOrderCriteria.Size.toString(), apiToken);
 
         getVersionFilesResponseSizeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3378,7 +3380,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
                 .body("data[4].label", equalTo(testFileName4));
 
         // Test Type order criteria
-        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileMetadatasOrderCriteria.Type.toString(), apiToken);
+        Response getVersionFilesResponseTypeCriteria = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, null, DatasetVersionFilesServiceBean.FileOrderCriteria.Type.toString(), apiToken);
 
         getVersionFilesResponseTypeCriteria.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3427,7 +3429,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
         restrictFileResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Restricted.toString(), null, null, null, null, apiToken);
+        Response getVersionFilesResponseRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, FileSearchCriteria.FileAccessStatus.Restricted.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponseRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3452,7 +3454,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
         createActiveFileEmbargoResponse.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenPublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponseEmbargoedThenPublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3461,7 +3463,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
         fileMetadatasCount = getVersionFilesResponseEmbargoedThenPublic.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
 
-        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, null, apiToken);
+        Response getVersionFilesResponseEmbargoedThenRestricted = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, FileSearchCriteria.FileAccessStatus.EmbargoedThenRestricted.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponseEmbargoedThenRestricted.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3471,7 +3473,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
         assertEquals(1, fileMetadatasCount);
 
         // Test Access Status Public
-        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString(), null, null, null, null, apiToken);
+        Response getVersionFilesResponsePublic = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, FileSearchCriteria.FileAccessStatus.Public.toString(), null, null, null, null, apiToken);
 
         getVersionFilesResponsePublic.then().assertThat()
                 .statusCode(OK.getStatusCode())
@@ -3569,7 +3571,7 @@ public void getVersionFileCounts() throws IOException {
         assertEquals(2, responseCountPerContentTypeMap.get("image/png"));
         assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
         assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
-        assertEquals(3, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString()));
-        assertEquals(1, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString()));
+        assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
+        assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
     }
 }

From 94fe70952ed46fd833f5948dfd9dc6ba92b95f6a Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 6 Oct 2023 14:49:55 +0100
Subject: [PATCH 531/922] Fixed: missing @Test annotation added to
 getVersionFilesIT

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index b9f09cc7c07..cdd1b4dff2b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3267,6 +3267,7 @@ public void getDatasetVersionCitation() {
                 .body("data.message", containsString("DRAFT VERSION"));
     }
 
+    @Test
     public void getVersionFiles() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());

From 690ac1e96a2717774e04aefb11603ae126005559 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 6 Oct 2023 15:29:45 +0100
Subject: [PATCH 532/922] Added: file search criteria params to
 getVersionFileCounts API endpoint (Pending IT to be added)

---
 .../DatasetVersionFilesServiceBean.java       | 99 ++++++++++---------
 .../harvard/iq/dataverse/api/Datasets.java    | 48 ++++++---
 2 files changed, 89 insertions(+), 58 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index a436b10d340..9afd0513b62 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -54,26 +54,32 @@ public enum FileOrderCriteria {
      * Given a DatasetVersion, returns its total file metadata count
      *
      * @param datasetVersion the DatasetVersion to access
+     * @param searchCriteria for counting only files matching this criteria
      * @return long value of total file metadata count
      */
-    public long getFileMetadataCount(DatasetVersion datasetVersion) {
+    public long getFileMetadataCount(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).stream().count();
+        JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
+        return baseQuery.stream().count();
     }
 
     /**
      * Given a DatasetVersion, returns its file metadata count per content type
      *
      * @param datasetVersion the DatasetVersion to access
+     * @param searchCriteria for counting only files matching this criteria
      * @return Map<String, Long> of file metadata counts per content type
      */
-    public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion) {
+    public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        List<Tuple> contentTypeOccurrences = queryFactory
+        JPAQuery<Tuple> baseQuery = queryFactory
                 .select(fileMetadata.dataFile.contentType, fileMetadata.count())
                 .from(fileMetadata)
                 .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                .groupBy(fileMetadata.dataFile.contentType).fetch();
+                .groupBy(fileMetadata.dataFile.contentType);
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
+        List<Tuple> contentTypeOccurrences = baseQuery.fetch();
         Map<String, Long> result = new HashMap<>();
         for (Tuple occurrence : contentTypeOccurrences) {
             result.put(occurrence.get(fileMetadata.dataFile.contentType), occurrence.get(fileMetadata.count()));
@@ -85,15 +91,18 @@ public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datas
      * Given a DatasetVersion, returns its file metadata count per category name
      *
      * @param datasetVersion the DatasetVersion to access
+     * @param searchCriteria for counting only files matching this criteria
      * @return Map<String, Long> of file metadata counts per category name
      */
-    public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion) {
+    public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        List<Tuple> categoryNameOccurrences = queryFactory
+        JPAQuery<Tuple> baseQuery = queryFactory
                 .select(dataFileCategory.name, fileMetadata.count())
                 .from(dataFileCategory, fileMetadata)
                 .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.fileCategories.contains(dataFileCategory)))
-                .groupBy(dataFileCategory.name).fetch();
+                .groupBy(dataFileCategory.name);
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
+        List<Tuple> categoryNameOccurrences = baseQuery.fetch();
         Map<String, Long> result = new HashMap<>();
         for (Tuple occurrence : categoryNameOccurrences) {
             result.put(occurrence.get(dataFileCategory.name), occurrence.get(fileMetadata.count()));
@@ -105,14 +114,15 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
      * Given a DatasetVersion, returns its file metadata count per FileAccessStatus
      *
      * @param datasetVersion the DatasetVersion to access
+     * @param searchCriteria for counting only files matching this criteria
      * @return Map<FileAccessStatus, Long> of file metadata counts per FileAccessStatus
      */
-    public Map<FileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) {
+    public Map<FileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         Map<FileAccessStatus, Long> allCounts = new HashMap<>();
-        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.Public);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.Restricted);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.EmbargoedThenPublic);
-        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.EmbargoedThenRestricted);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.Public, searchCriteria);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.Restricted, searchCriteria);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.EmbargoedThenPublic, searchCriteria);
+        addAccessStatusCountToTotal(datasetVersion, allCounts, FileAccessStatus.EmbargoedThenRestricted, searchCriteria);
         return allCounts;
     }
 
@@ -128,54 +138,31 @@ public Map<FileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVe
      */
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileSearchCriteria searchCriteria, FileOrderCriteria orderCriteria) {
         JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
-
-        String contentType = searchCriteria.getContentType();
-        if (contentType != null) {
-            baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
-        }
-        FileAccessStatus accessStatus = searchCriteria.getAccessStatus();
-        if (accessStatus != null) {
-            baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
-        }
-        String categoryName = searchCriteria.getCategoryName();
-        if (categoryName != null) {
-            baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
-        }
-        String tabularTagName = searchCriteria.getTabularTagName();
-        if (tabularTagName != null) {
-            baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
-        }
-        String searchText = searchCriteria.getSearchText();
-        if (searchText != null && !searchText.isEmpty()) {
-            searchText = searchText.trim().toLowerCase();
-            baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
-        }
-
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
         applyOrderCriteriaToGetFileMetadatasQuery(baseQuery, orderCriteria);
-
         if (limit != null) {
             baseQuery.limit(limit);
         }
         if (offset != null) {
             baseQuery.offset(offset);
         }
-
         return baseQuery.fetch();
     }
 
-    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<FileAccessStatus, Long> totalCounts, FileAccessStatus dataFileAccessStatus) {
-        long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus);
+    private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<FileAccessStatus, Long> totalCounts, FileAccessStatus dataFileAccessStatus, FileSearchCriteria searchCriteria) {
+        long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus, searchCriteria);
         if (fileMetadataCount > 0) {
             totalCounts.put(dataFileAccessStatus, fileMetadataCount);
         }
     }
 
-    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, FileAccessStatus accessStatus) {
+    private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, FileAccessStatus accessStatus, FileSearchCriteria searchCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        return queryFactory
+        JPAQuery<FileMetadata> baseQuery = queryFactory
                 .selectFrom(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus)))
-                .stream().count();
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus)));
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
+        return baseQuery.stream().count();
     }
 
     private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileOrderCriteria orderCriteria) {
@@ -211,6 +198,30 @@ private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAcces
         return accessStatusExpression;
     }
 
+    private void applyFileSearchCriteriaToQuery(JPAQuery<?> baseQuery, FileSearchCriteria searchCriteria) {
+        String contentType = searchCriteria.getContentType();
+        if (contentType != null) {
+            baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
+        }
+        FileAccessStatus accessStatus = searchCriteria.getAccessStatus();
+        if (accessStatus != null) {
+            baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
+        }
+        String categoryName = searchCriteria.getCategoryName();
+        if (categoryName != null) {
+            baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
+        }
+        String tabularTagName = searchCriteria.getTabularTagName();
+        if (tabularTagName != null) {
+            baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
+        }
+        String searchText = searchCriteria.getSearchText();
+        if (searchText != null && !searchText.isEmpty()) {
+            searchText = searchText.trim().toLowerCase();
+            baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
+        }
+    }
+
     private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileOrderCriteria orderCriteria) {
         DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
         switch (orderCriteria) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 14fd1b2453c..ac32454c950 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -512,19 +512,18 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
             }
-            FileSearchCriteria.FileAccessStatus dataFileAccessStatus;
+            FileSearchCriteria fileSearchCriteria;
             try {
-                dataFileAccessStatus = accessStatus != null ? FileSearchCriteria.FileAccessStatus.valueOf(accessStatus) : null;
+                fileSearchCriteria = new FileSearchCriteria(
+                        contentType,
+                        accessStatus != null ? FileSearchCriteria.FileAccessStatus.valueOf(accessStatus) : null,
+                        categoryName,
+                        tabularTagName,
+                        searchText
+                );
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
             }
-            FileSearchCriteria fileSearchCriteria = new FileSearchCriteria(
-                    contentType,
-                    dataFileAccessStatus,
-                    categoryName,
-                    tabularTagName,
-                    searchText
-            );
             return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, fileSearchCriteria, fileOrderCriteria)));
         }, getRequestUser(crc));
     }
@@ -532,14 +531,35 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/files/counts")
-    public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+    public Response getVersionFileCounts(@Context ContainerRequestContext crc,
+                                         @PathParam("id") String datasetId,
+                                         @PathParam("versionId") String versionId,
+                                         @QueryParam("contentType") String contentType,
+                                         @QueryParam("accessStatus") String accessStatus,
+                                         @QueryParam("categoryName") String categoryName,
+                                         @QueryParam("tabularTagName") String tabularTagName,
+                                         @QueryParam("searchText") String searchText,
+                                         @Context UriInfo uriInfo,
+                                         @Context HttpHeaders headers) {
         return response(req -> {
+            FileSearchCriteria fileSearchCriteria;
+            try {
+                fileSearchCriteria = new FileSearchCriteria(
+                        contentType,
+                        accessStatus != null ? FileSearchCriteria.FileAccessStatus.valueOf(accessStatus) : null,
+                        categoryName,
+                        tabularTagName,
+                        searchText
+                );
+            } catch (IllegalArgumentException e) {
+                return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
+            }
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
-            jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion));
-            jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion)));
-            jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion)));
-            jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion)));
+            jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion, fileSearchCriteria));
+            jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion, fileSearchCriteria)));
+            jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion, fileSearchCriteria)));
+            jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion, fileSearchCriteria)));
             return ok(jsonObjectBuilder);
         }, getRequestUser(crc));
     }

From a0870b8554c709f25fb3bc47e04f58e08e951f2f Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 6 Oct 2023 15:35:17 +0100
Subject: [PATCH 533/922] Refactor: using Bundle.properties string for bad
 request errors in getVersionFiles and getVersionFileCounts API endpoints

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 6 +++---
 src/main/java/propertyFiles/Bundle.properties            | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index ac32454c950..f7a4b1d0d25 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -510,7 +510,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
             try {
                 fileOrderCriteria = orderCriteria != null ? DatasetVersionFilesServiceBean.FileOrderCriteria.valueOf(orderCriteria) : DatasetVersionFilesServiceBean.FileOrderCriteria.NameAZ;
             } catch (IllegalArgumentException e) {
-                return error(Response.Status.BAD_REQUEST, "Invalid order criteria: " + orderCriteria);
+                return badRequest(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.order.criteria", List.of(orderCriteria)));
             }
             FileSearchCriteria fileSearchCriteria;
             try {
@@ -522,7 +522,7 @@ public Response getVersionFiles(@Context ContainerRequestContext crc,
                         searchText
                 );
             } catch (IllegalArgumentException e) {
-                return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
+                return badRequest(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.access.status", List.of(accessStatus)));
             }
             return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, fileSearchCriteria, fileOrderCriteria)));
         }, getRequestUser(crc));
@@ -552,7 +552,7 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc,
                         searchText
                 );
             } catch (IllegalArgumentException e) {
-                return error(Response.Status.BAD_REQUEST, "Invalid access status: " + accessStatus);
+                return badRequest(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.access.status", List.of(accessStatus)));
             }
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 7b4befcca36..3128106d38f 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2646,6 +2646,8 @@ datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymi
 datasets.api.creationdate=Date Created
 datasets.api.modificationdate=Last Modified Date
 datasets.api.curationstatus=Curation Status
+datasets.api.version.files.invalid.order.criteria=Invalid order criteria: {0}
+datasets.api.version.files.invalid.access.status=Invalid access status: {0}
 
 
 #Dataverses.java

From 2abb36fc2f24e78ca75ebe0cbfc0a84a1345af26 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 6 Oct 2023 17:00:56 +0100
Subject: [PATCH 534/922] Added: IT for getVersionFileCounts with criteria

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 127 +++++++++++++++++-
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  22 ++-
 2 files changed, 143 insertions(+), 6 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 5d1a89aa555..433628685b2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3484,6 +3484,13 @@ public void getVersionFiles() throws IOException, InterruptedException {
         fileMetadatasCount = getVersionFilesResponsePublic.jsonPath().getList("data").size();
         assertEquals(3, fileMetadatasCount);
 
+        // Test invalid access status
+        String invalidStatus = "invalidStatus";
+        Response getVersionFilesResponseInvalidStatus = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, invalidStatus, null, null, null, null, apiToken);
+        getVersionFilesResponseInvalidStatus.then().assertThat()
+                .statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", equalTo(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.access.status", List.of(invalidStatus))));
+
         // Test Search Text
         Response getVersionFilesResponseSearchText = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, null, null, "test_1", null, apiToken);
 
@@ -3519,7 +3526,7 @@ public void getVersionFiles() throws IOException, InterruptedException {
     }
 
     @Test
-    public void getVersionFileCounts() throws IOException {
+    public void getVersionFileCounts() throws IOException, InterruptedException {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
@@ -3557,8 +3564,10 @@ public void getVersionFileCounts() throws IOException {
         Response createFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(dataFileId), activeEmbargoDate, apiToken);
         createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode());
 
-        // Getting the file counts and assert each count
-        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken);
+        String testDatasetVersion = ":latest";
+
+        // Getting the file counts without criteria and assert each count is correct
+        Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, null, null, null, null, apiToken);
 
         getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
 
@@ -3570,8 +3579,120 @@ public void getVersionFileCounts() throws IOException {
         assertEquals(4, (Integer) responseJsonPath.get("data.total"));
         assertEquals(2, responseCountPerContentTypeMap.get("image/png"));
         assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
+        assertEquals(2, responseCountPerContentTypeMap.size());
+        assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(2, responseCountPerAccessStatusMap.size());
+        assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
+        assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
+
+        // Test content type criteria
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, "image/png", null, null, null, null, apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(2, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(2, responseCountPerContentTypeMap.get("image/png"));
+        assertEquals(1, responseCountPerContentTypeMap.size());
+        assertEquals(1, responseCountPerCategoryNameMap.size());
         assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(2, responseCountPerAccessStatusMap.size());
+        assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
+        assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
+
+        // Test access status criteria
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, FileSearchCriteria.FileAccessStatus.Public.toString(), null, null, null, apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(3, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(1, responseCountPerContentTypeMap.get("image/png"));
+        assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
+        assertEquals(2, responseCountPerContentTypeMap.size());
+        assertEquals(0, responseCountPerCategoryNameMap.size());
+        assertEquals(1, responseCountPerAccessStatusMap.size());
         assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
+
+        // Test invalid access status
+        String invalidStatus = "invalidStatus";
+        Response getVersionFilesResponseInvalidStatus = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, invalidStatus, null, null, null, apiToken);
+        getVersionFilesResponseInvalidStatus.then().assertThat()
+                .statusCode(BAD_REQUEST.getStatusCode())
+                .body("message", equalTo(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.access.status", List.of(invalidStatus))));
+
+        // Test category name criteria
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, null, "testCategory", null, null, apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(1, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(1, responseCountPerContentTypeMap.get("image/png"));
+        assertEquals(1, responseCountPerContentTypeMap.size());
+        assertEquals(1, responseCountPerCategoryNameMap.size());
+        assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(1, responseCountPerAccessStatusMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
+
+        // Test search text criteria
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, null, null, null, "test", apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(3, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(1, responseCountPerContentTypeMap.get("image/png"));
+        assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
+        assertEquals(2, responseCountPerContentTypeMap.size());
+        assertEquals(0, responseCountPerCategoryNameMap.size());
+        assertEquals(1, responseCountPerAccessStatusMap.size());
+        assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
+
+        // Test tabular tag name criteria
+        String pathToTabularTestFile = "src/test/resources/tab/test.tab";
+        Response uploadTabularFileResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTabularTestFile, Json.createObjectBuilder().build(), apiToken);
+        uploadTabularFileResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        String tabularFileId = uploadTabularFileResponse.getBody().jsonPath().getString("data.files[0].dataFile.id");
+
+        // Ensure tabular file is ingested
+        sleep(2000);
+
+        String tabularTagName = "Survey";
+        Response setFileTabularTagsResponse = UtilIT.setFileTabularTags(tabularFileId, apiToken, List.of(tabularTagName));
+        setFileTabularTagsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, null, null, tabularTagName, null, apiToken);
+
+        getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        responseJsonPath = getVersionFileCountsResponse.jsonPath();
+        responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
+        responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
+
+        assertEquals(1, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(1, responseCountPerContentTypeMap.get("text/tab-separated-values"));
+        assertEquals(1, responseCountPerContentTypeMap.size());
+        assertEquals(0, responseCountPerCategoryNameMap.size());
+        assertEquals(1, responseCountPerAccessStatusMap.size());
+        assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 8e333451c8d..6d0f0bfa752 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3358,10 +3358,26 @@ static Response createFileEmbargo(Integer datasetId, Integer fileId, String date
                 .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
     }
 
-    static Response getVersionFileCounts(Integer datasetId, String version, String apiToken) {
-        return given()
+    static Response getVersionFileCounts(Integer datasetId, String version, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, String apiToken) {
+        RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
+                .contentType("application/json");
+        if (contentType != null) {
+            requestSpecification = requestSpecification.queryParam("contentType", contentType);
+        }
+        if (accessStatus != null) {
+            requestSpecification = requestSpecification.queryParam("accessStatus", accessStatus);
+        }
+        if (categoryName != null) {
+            requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
+        }
+        if (tabularTagName != null) {
+            requestSpecification = requestSpecification.queryParam("tabularTagName", tabularTagName);
+        }
+        if (searchText != null) {
+            requestSpecification = requestSpecification.queryParam("searchText", searchText);
+        }
+        return requestSpecification.get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
     }
 
     static Response setFileCategories(String dataFileId, String apiToken, List<String> categories) {

From 65df3d0f4bca41598dcc5cad741779d7d8fd5716 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 9 Oct 2023 09:36:40 +0100
Subject: [PATCH 535/922] Added: count per tabular tag name to
 getVersionFileCounts API endpoint

---
 .../DatasetVersionFilesServiceBean.java       | 23 +++++++++++++++++++
 .../harvard/iq/dataverse/api/Datasets.java    |  1 +
 .../iq/dataverse/util/json/JsonPrinter.java   |  8 +++++++
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 13 +++++++++++
 4 files changed, 45 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 9afd0513b62..b6b095f58dd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -110,6 +110,29 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
         return result;
     }
 
+    /**
+     * Given a DatasetVersion, returns its file metadata count per DataFileTag.TagType
+     *
+     * @param datasetVersion the DatasetVersion to access
+     * @param searchCriteria for counting only files matching this criteria
+     * @return Map<DataFileTag.TagType, Long> of file metadata counts per DataFileTag.TagType
+     */
+    public Map<DataFileTag.TagType, Long> getFileMetadataCountPerTabularTagName(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
+        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
+        JPAQuery<Tuple> baseQuery = queryFactory
+                .select(dataFileTag.type, fileMetadata.count())
+                .from(dataFileTag, fileMetadata)
+                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)))
+                .groupBy(dataFileTag.type);
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
+        List<Tuple> tagNameOccurrences = baseQuery.fetch();
+        Map<DataFileTag.TagType, Long> result = new HashMap<>();
+        for (Tuple occurrence : tagNameOccurrences) {
+            result.put(occurrence.get(dataFileTag.type), occurrence.get(fileMetadata.count()));
+        }
+        return result;
+    }
+
     /**
      * Given a DatasetVersion, returns its file metadata count per FileAccessStatus
      *
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index f7a4b1d0d25..26d4dd01cf5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -559,6 +559,7 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc,
             jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion, fileSearchCriteria));
             jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion, fileSearchCriteria)));
             jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion, fileSearchCriteria)));
+            jsonObjectBuilder.add("perTabularTagName", jsonFileCountPerTabularTagNameMap(datasetVersionFilesServiceBean.getFileMetadataCountPerTabularTagName(datasetVersion, fileSearchCriteria)));
             jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion, fileSearchCriteria)));
             return ok(jsonObjectBuilder);
         }, getRequestUser(crc));
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 70840c7502f..6fe1ca87028 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -1115,6 +1115,14 @@ public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map<FileSearchCr
         return jsonObjectBuilder;
     }
 
+    public static JsonObjectBuilder jsonFileCountPerTabularTagNameMap(Map<DataFileTag.TagType, Long> map) {
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        for (Map.Entry<DataFileTag.TagType, Long> mapEntry : map.entrySet()) {
+            jsonObjectBuilder.add(mapEntry.getKey().toString(), mapEntry.getValue());
+        }
+        return jsonObjectBuilder;
+    }
+
     public static Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder> toJsonArray() {
         return new Collector<JsonObjectBuilder, ArrayList<JsonObjectBuilder>, JsonArrayBuilder>() {
 
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 433628685b2..53546133b27 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3574,6 +3574,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath();
         LinkedHashMap<String, Integer> responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         LinkedHashMap<String, Integer> responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        LinkedHashMap<String, Integer> responseCountPerTabularTagNameMap = responseJsonPath.get("data.perTabularTagName");
         LinkedHashMap<String, Integer> responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
 
         assertEquals(4, (Integer) responseJsonPath.get("data.total"));
@@ -3581,6 +3582,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
         assertEquals(2, responseCountPerContentTypeMap.size());
         assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(0, responseCountPerTabularTagNameMap.size());
         assertEquals(2, responseCountPerAccessStatusMap.size());
         assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
@@ -3593,6 +3595,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         responseJsonPath = getVersionFileCountsResponse.jsonPath();
         responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerTabularTagNameMap = responseJsonPath.get("data.perTabularTagName");
         responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
 
         assertEquals(2, (Integer) responseJsonPath.get("data.total"));
@@ -3600,6 +3603,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         assertEquals(1, responseCountPerContentTypeMap.size());
         assertEquals(1, responseCountPerCategoryNameMap.size());
         assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(0, responseCountPerTabularTagNameMap.size());
         assertEquals(2, responseCountPerAccessStatusMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
@@ -3612,6 +3616,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         responseJsonPath = getVersionFileCountsResponse.jsonPath();
         responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerTabularTagNameMap = responseJsonPath.get("data.perTabularTagName");
         responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
 
         assertEquals(3, (Integer) responseJsonPath.get("data.total"));
@@ -3619,6 +3624,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
         assertEquals(2, responseCountPerContentTypeMap.size());
         assertEquals(0, responseCountPerCategoryNameMap.size());
+        assertEquals(0, responseCountPerTabularTagNameMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.size());
         assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
 
@@ -3637,6 +3643,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         responseJsonPath = getVersionFileCountsResponse.jsonPath();
         responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerTabularTagNameMap = responseJsonPath.get("data.perTabularTagName");
         responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
 
         assertEquals(1, (Integer) responseJsonPath.get("data.total"));
@@ -3644,6 +3651,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         assertEquals(1, responseCountPerContentTypeMap.size());
         assertEquals(1, responseCountPerCategoryNameMap.size());
         assertEquals(1, responseCountPerCategoryNameMap.get(testCategory));
+        assertEquals(0, responseCountPerTabularTagNameMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
 
@@ -3655,6 +3663,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         responseJsonPath = getVersionFileCountsResponse.jsonPath();
         responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerTabularTagNameMap = responseJsonPath.get("data.perTabularTagName");
         responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
 
         assertEquals(3, (Integer) responseJsonPath.get("data.total"));
@@ -3662,6 +3671,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         assertEquals(2, responseCountPerContentTypeMap.get("text/plain"));
         assertEquals(2, responseCountPerContentTypeMap.size());
         assertEquals(0, responseCountPerCategoryNameMap.size());
+        assertEquals(0, responseCountPerTabularTagNameMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.size());
         assertEquals(3, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
 
@@ -3686,12 +3696,15 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         responseJsonPath = getVersionFileCountsResponse.jsonPath();
         responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType");
         responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName");
+        responseCountPerTabularTagNameMap = responseJsonPath.get("data.perTabularTagName");
         responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus");
 
         assertEquals(1, (Integer) responseJsonPath.get("data.total"));
         assertEquals(1, responseCountPerContentTypeMap.get("text/tab-separated-values"));
         assertEquals(1, responseCountPerContentTypeMap.size());
         assertEquals(0, responseCountPerCategoryNameMap.size());
+        assertEquals(1, responseCountPerTabularTagNameMap.size());
+        assertEquals(1, responseCountPerTabularTagNameMap.get(tabularTagName));
         assertEquals(1, responseCountPerAccessStatusMap.size());
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.Public.toString()));
     }

From 98a444c2108395fc562e0159d554ce1f9968686e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 9 Oct 2023 09:45:15 +0100
Subject: [PATCH 536/922] Added: docs for extended getVersionFileCounts
 endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 52 +++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 97b41ffa98a..f05c4d42073 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1046,6 +1046,7 @@ The returned file counts are based on different criteria:
 - Total (The total file count)
 - Per content type
 - Per category name
+- Per tabular tag name
 - Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic)
 
 .. code-block:: bash
@@ -1062,6 +1063,57 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts"
 
+Category name filtering is optionally supported. To return counts only for files to which the requested category has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?categoryName=Data"
+
+Tabular tag name filtering is also optionally supported. To return counts only for files to which the requested tabular tag has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?tabularTagName=Survey"
+
+Content type filtering is also optionally supported. To return counts only for files matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?contentType=image/png"
+
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return counts only for files that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?searchText=word"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?accessStatus=Public"
+
+Please note that filtering values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+Keep in mind that you can combine all of the above query params depending on the results you are looking for.
+
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 7d0501cdc2982e591d99eab29b9569d2880ebf30 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 9 Oct 2023 09:50:30 +0100
Subject: [PATCH 537/922] Added: #9907 release notes

---
 .../9907-files-api-counts-with-criteria.md            | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 doc/release-notes/9907-files-api-counts-with-criteria.md

diff --git a/doc/release-notes/9907-files-api-counts-with-criteria.md b/doc/release-notes/9907-files-api-counts-with-criteria.md
new file mode 100644
index 00000000000..07cd23daad0
--- /dev/null
+++ b/doc/release-notes/9907-files-api-counts-with-criteria.md
@@ -0,0 +1,11 @@
+Extended the getVersionFileCounts endpoint (/api/datasets/{id}/versions/{versionId}/files/counts) to support filtering by criteria.
+
+In particular, the endpoint now accepts the following optional criteria query parameters:
+
+- contentType
+- accessStatus
+- categoryName
+- tabularTagName
+- searchText
+
+This filtering criteria is the same as the one for the getVersionFiles endpoint.

From 35eeed53cefe427df8684ca8c20046be2b2a45f2 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 9 Oct 2023 10:07:53 +0100
Subject: [PATCH 538/922] Refactor: using variable instead of repeated string
 in IT

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 53546133b27..06d0bed14c0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3636,7 +3636,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
                 .body("message", equalTo(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.access.status", List.of(invalidStatus))));
 
         // Test category name criteria
-        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, null, "testCategory", null, null, apiToken);
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, null, null, testCategory, null, null, apiToken);
 
         getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
 

From 1e8b735ca1baba7c12bac0737cfc88eedc084ec3 Mon Sep 17 00:00:00 2001
From: mr-loop-1 <loopolecreator1@gmail.com>
Date: Mon, 9 Oct 2023 14:47:26 +0530
Subject: [PATCH 539/922] #9412 added markdown in external tools guide

---
 .../source/_static/admin/dataverse-external-tools.tsv           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
index 8543300dd2c..f8bf5fc73d9 100644
--- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
+++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
@@ -2,6 +2,6 @@ Tool	Type	Scope	Description
 Data Explorer	explore	file	"A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse."
 Whole Tale	explore	dataset	"A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide <https://wholetale.readthedocs.io/en/stable/users_guide/integration.html>`_."
 Binder	explore	dataset	Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. `Installation instructions <https://github.com/data-exp-lab/girder_ythub/issues/10>`_ are in the Data Exploration Lab girder_ythub project.
-File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
+File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, markdown (md), text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
 Data Curation Tool	configure	file	"A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions."
 Ask the Data	query	file	Ask the Data is an experimental tool that allows you ask natural language questions about the data contained in Dataverse tables (tabular data). See the README.md file at https://github.com/IQSS/askdataverse/tree/main/askthedata for the instructions on adding Ask the Data to your Dataverse installation. 

From b32d51fab1b78ff9316476f76c549163c4b3e7ba Mon Sep 17 00:00:00 2001
From: mr-loop-1 <loopolecreator1@gmail.com>
Date: Mon, 9 Oct 2023 14:50:28 +0530
Subject: [PATCH 540/922] #9412 added markdown to gile previews list

---
 doc/sphinx-guides/source/user/dataset-management.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst
index 3b5b4ec6ba8..c41ca40dd36 100755
--- a/doc/sphinx-guides/source/user/dataset-management.rst
+++ b/doc/sphinx-guides/source/user/dataset-management.rst
@@ -200,6 +200,7 @@ Previewers are available for the following file types:
 
 - Text
 - PDF
+- Markdown (MD)
 - Tabular (CSV, Excel, etc., see :doc:`tabulardataingest/index`)
 - Code (R, etc.)
 - Images (PNG, GIF, JPG)

From dea8bf7636bc396d51773bc481135dab5f1a7679 Mon Sep 17 00:00:00 2001
From: mr-loop-1 <loopolecreator1@gmail.com>
Date: Mon, 9 Oct 2023 14:51:13 +0530
Subject: [PATCH 541/922] #9412 removed file extension markdown

---
 .../source/_static/admin/dataverse-external-tools.tsv           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
index f8bf5fc73d9..a13dea923e4 100644
--- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
+++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
@@ -2,6 +2,6 @@ Tool	Type	Scope	Description
 Data Explorer	explore	file	"A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse."
 Whole Tale	explore	dataset	"A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide <https://wholetale.readthedocs.io/en/stable/users_guide/integration.html>`_."
 Binder	explore	dataset	Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. `Installation instructions <https://github.com/data-exp-lab/girder_ythub/issues/10>`_ are in the Data Exploration Lab girder_ythub project.
-File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, markdown (md), text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
+File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, markdown, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
 Data Curation Tool	configure	file	"A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions."
 Ask the Data	query	file	Ask the Data is an experimental tool that allows you ask natural language questions about the data contained in Dataverse tables (tabular data). See the README.md file at https://github.com/IQSS/askdataverse/tree/main/askthedata for the instructions on adding Ask the Data to your Dataverse installation. 

From 5a3b7853607d0a995ad9cbdbbcf402114f2a70b8 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 9 Oct 2023 08:53:43 -0400
Subject: [PATCH 542/922] add release note for markdown previewer #9412

---
 doc/release-notes/9412-markdown-previewer.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/release-notes/9412-markdown-previewer.md

diff --git a/doc/release-notes/9412-markdown-previewer.md b/doc/release-notes/9412-markdown-previewer.md
new file mode 100644
index 00000000000..8faa2679fb0
--- /dev/null
+++ b/doc/release-notes/9412-markdown-previewer.md
@@ -0,0 +1 @@
+There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews

From cc117bd4396e18f5680f34488928bb7a009b8bf0 Mon Sep 17 00:00:00 2001
From: Abdul Samad <62374784+mr-loop-1@users.noreply.github.com>
Date: Mon, 9 Oct 2023 19:03:00 +0530
Subject: [PATCH 543/922] remove extension after markdown

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/user/dataset-management.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst
index c41ca40dd36..1e8ea897032 100755
--- a/doc/sphinx-guides/source/user/dataset-management.rst
+++ b/doc/sphinx-guides/source/user/dataset-management.rst
@@ -200,7 +200,7 @@ Previewers are available for the following file types:
 
 - Text
 - PDF
-- Markdown (MD)
+- Markdown
 - Tabular (CSV, Excel, etc., see :doc:`tabulardataingest/index`)
 - Code (R, etc.)
 - Images (PNG, GIF, JPG)

From 44b015f375fc92505def8ef2e2475950a3818d4e Mon Sep 17 00:00:00 2001
From: Abdul Samad <62374784+mr-loop-1@users.noreply.github.com>
Date: Mon, 9 Oct 2023 19:03:32 +0530
Subject: [PATCH 544/922] Capitalise Markdown

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 .../source/_static/admin/dataverse-external-tools.tsv           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
index a13dea923e4..4f4c29d0670 100644
--- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
+++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
@@ -2,6 +2,6 @@ Tool	Type	Scope	Description
 Data Explorer	explore	file	"A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse."
 Whole Tale	explore	dataset	"A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide <https://wholetale.readthedocs.io/en/stable/users_guide/integration.html>`_."
 Binder	explore	dataset	Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. `Installation instructions <https://github.com/data-exp-lab/girder_ythub/issues/10>`_ are in the Data Exploration Lab girder_ythub project.
-File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, markdown, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
+File Previewers	explore	file	"A set of tools that display the content of files - including audio, html, `Hypothes.is <https://hypothes.is/>`_ annotations, images, PDF, Markdown, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
 Data Curation Tool	configure	file	"A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions."
 Ask the Data	query	file	Ask the Data is an experimental tool that allows you ask natural language questions about the data contained in Dataverse tables (tabular data). See the README.md file at https://github.com/IQSS/askdataverse/tree/main/askthedata for the instructions on adding Ask the Data to your Dataverse installation. 

From e1acdd328fa4a6ca6624522e21806c7d2a779ef9 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Oct 2023 14:49:09 +0200
Subject: [PATCH 545/922] test(oidc): deactivate test when no Docker available
 #9974

As many of IQSS and external devs might not have Docker available,
let's deactivate any Testcontainers tests in these cases.
---
 .../oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
index 5968cf3eaeb..ee6823ef98a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/oidc/OIDCAuthenticationProviderFactoryIT.java
@@ -54,7 +54,7 @@
 
 @Tag(Tags.INTEGRATION_TEST)
 @Tag(Tags.USES_TESTCONTAINERS)
-@Testcontainers
+@Testcontainers(disabledWithoutDocker = true)
 @ExtendWith(MockitoExtension.class)
 // NOTE: order is important here - Testcontainers must be first, otherwise it's not ready when we call getAuthUrl()
 @LocalJvmSettings

From 2aa7a471249cb129aeef13d6301f10ddb43506b7 Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <o.bertuch@fz-juelich.de>
Date: Tue, 10 Oct 2023 14:52:43 +0200
Subject: [PATCH 546/922] doc(testing): change docs for TC ITs to disable when
 no Docker #9974

---
 doc/sphinx-guides/source/developers/testing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 4691aca3aad..dab8110b20b 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -316,7 +316,7 @@ Please make sure to:
    .. code:: java
 
        /** A very minimal example for a Testcontainers integration test class. */
-       @Testcontainers
+       @Testcontainers(disabledWithoutDocker = true)
        @Tag(edu.harvard.iq.dataverse.util.testing.Tags.INTEGRATION_TEST)
        @Tag(edu.harvard.iq.dataverse.util.testing.Tags.USES_TESTCONTAINERS)
        class MyExampleIT { /* ... */ }

From ed291936810a46e260df9809def80b2d2c5b50dc Mon Sep 17 00:00:00 2001
From: Oliver Bertuch <poikilotherm@users.noreply.github.com>
Date: Tue, 10 Oct 2023 20:04:11 +0200
Subject: [PATCH 547/922] style(ct): remove empty lines from configbaker
 Dockerfile

---
 modules/container-configbaker/Dockerfile | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile
index 2975b043213..9b98334d72b 100644
--- a/modules/container-configbaker/Dockerfile
+++ b/modules/container-configbaker/Dockerfile
@@ -40,8 +40,6 @@ COPY maven/solr/*.xml ${SOLR_TEMPLATE}/conf/
 RUN rm ${SOLR_TEMPLATE}/conf/managed-schema.xml
 
 
-
-
 # Set the entrypoint to tini (as a process supervisor)
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 # By default run a script that will print a help message and terminate

From e89e2aaeb32f983462ea11b64eceab6ddc926eb7 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 11 Oct 2023 10:47:00 -0400
Subject: [PATCH 548/922] #9507 revert to use dataverse in bundle

---
 src/main/java/propertyFiles/Bundle.properties | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 4964dac78a4..e3dbdc144f1 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -875,22 +875,22 @@ dataverse.nopublished=No Published Dataverses
 dataverse.nopublished.tip=In order to use this feature you must have at least one published dataverse.
 dataverse.contact=Email Dataverse Contact
 dataverse.link=Link Collection
-dataverse.link.btn.tip=Link to Your Collection
-dataverse.link.yourDataverses=Your Collection
-dataverse.link.yourDataverses.inputPlaceholder=Enter Collection Name
-dataverse.link.save=Save Linked collection
-dataverse.link.dataverse.choose=Choose which of your collection you would like to link this collection to.
-dataverse.link.dataset.choose=Enter the name of the collection you would like to link this dataset to. If you need to remove this link in the future, please contact {0}.
-dataverse.link.dataset.none=No linkable collections available.
-dataverse.link.no.choice=You have one collection you can add linked collection and datasets in.
-dataverse.link.no.linkable=To be able to link a collection or dataset, you need to have your own collection. Create a collection to get started.
-dataverse.link.no.linkable.remaining=You have already linked all of your eligible collections.
+dataverse.link.btn.tip=Link to Your Dataverse
+dataverse.link.yourDataverses=Your Dataverse
+dataverse.link.yourDataverses.inputPlaceholder=Enter Dataverse Name
+dataverse.link.save=Save Linked Dataverse
+dataverse.link.dataverse.choose=Choose which of your dataverses you would like to link this dataverse to.
+dataverse.link.dataset.choose=Enter the name of the dataverse you would like to link this dataset to. If you need to remove this link in the future, please contact {0}.
+dataverse.link.dataset.none=No linkable dataverses available.
+dataverse.link.no.choice=You have one dataverse you can add linked dataverses and datasets in.
+dataverse.link.no.linkable=To be able to link a dataverse or dataset, you need to have your own dataverse. Create a dataverse to get started.
+dataverse.link.no.linkable.remaining=You have already linked all of your eligible dataverses.
 dataverse.savedsearch.link=Link Search
 dataverse.savedsearch.searchquery=Search
 dataverse.savedsearch.filterQueries=Facets
 dataverse.savedsearch.save=Save Linked Search
-dataverse.savedsearch.dataverse.choose=Choose which of your collection you would like to link this search to.
-dataverse.savedsearch.no.choice=You have one collection to which you may add a saved search.
+dataverse.savedsearch.dataverse.choose=Choose which of your dataverses you would like to link this search to.
+dataverse.savedsearch.no.choice=You have one dataverse to which you may add a saved search.
 # Bundle file editors, please note that "dataverse.savedsearch.save.success" is used in a unit test
 dataverse.saved.search.success=The saved search has been successfully linked to {0}.
 dataverse.saved.search.failure=The saved search was not able to be linked.
@@ -2498,7 +2498,7 @@ dataset.registered=DatasetRegistered
 dataset.registered.msg=Your dataset is now registered.
 dataset.notlinked=DatasetNotLinked
 dataset.notlinked.msg=There was a problem linking this dataset to yours:
-dataset.linking.popop.already.linked.note=Note: This dataset is already linked to the following collection(s):  
+dataset.linking.popop.already.linked.note=Note: This dataset is already linked to the following dataverse(s):  
 datasetversion.archive.success=Archival copy of Version successfully submitted
 datasetversion.archive.failure=Error in submitting an archival copy 
 datasetversion.update.failure=Dataset Version Update failed. Changes are still in the DRAFT version.

From 617f36cd98b267bc99d53a7b69c21d96974ff4dc Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 11 Oct 2023 10:48:11 -0400
Subject: [PATCH 549/922] #9507 missed one

---
 src/main/java/propertyFiles/Bundle.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index e3dbdc144f1..89eabaeb0bf 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -874,7 +874,7 @@ dataverse.publish.header=Publish Dataverse
 dataverse.nopublished=No Published Dataverses
 dataverse.nopublished.tip=In order to use this feature you must have at least one published dataverse.
 dataverse.contact=Email Dataverse Contact
-dataverse.link=Link Collection
+dataverse.link=Link Dataverse
 dataverse.link.btn.tip=Link to Your Dataverse
 dataverse.link.yourDataverses=Your Dataverse
 dataverse.link.yourDataverses.inputPlaceholder=Enter Dataverse Name

From 18cdf133f49d597da6aea9d21385e45b77844ceb Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 11 Oct 2023 14:48:51 -0400
Subject: [PATCH 550/922] stripping more dead code in the version service bean
 (my experimental filemetadatas retrieval method, not directly used in the
 PR). (#9763)

---
 .../dataverse/DatasetVersionServiceBean.java  | 88 -------------------
 1 file changed, 88 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index 476a306e081..c2f9027a38a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -49,22 +49,6 @@ public class DatasetVersionServiceBean implements java.io.Serializable {
 
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL = "SELECT fm FROM FileMetadata fm"
-            + " WHERE fm.datasetVersion.id=:datasetVersionId"
-            + " ORDER BY fm.label";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE = "SELECT fm FROM FileMetadata fm, DvObject dvo"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = dvo.id"
-            + " ORDER BY CASE WHEN dvo.publicationDate IS NOT NULL THEN dvo.publicationDate ELSE dvo.createDate END";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE = "SELECT fm FROM FileMetadata fm, DataFile df"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = df.id"
-            + " ORDER BY df.filesize";
-    private static final String QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE = "SELECT fm FROM FileMetadata fm, DataFile df"
-            + " WHERE fm.datasetVersion.id = :datasetVersionId"
-            + " AND fm.dataFile.id = df.id"
-            + " ORDER BY df.contentType";
-
     @EJB
     DatasetServiceBean datasetService;
     
@@ -166,18 +150,6 @@ public DatasetVersion getDatasetVersion(){
         }                
     } // end RetrieveDatasetVersionResponse
 
-    /**
-     *  Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionServiceBean#getFileMetadatas}
-     */
-    public enum FileMetadatasOrderCriteria {
-        NameAZ,
-        NameZA,
-        Newest,
-        Oldest,
-        Size,
-        Type
-    }
-
     public DatasetVersion find(Object pk) {
         return em.find(DatasetVersion.class, pk);
     }
@@ -1287,64 +1259,4 @@ public List<DatasetVersion> getUnarchivedDatasetVersions(){
             return null;
         }
     } // end getUnarchivedDatasetVersions
-
-    /**
-     * Returns a FileMetadata list of files in the specified DatasetVersion
-     *
-     * @param datasetVersion the DatasetVersion to access
-     * @param limit for pagination, can be null
-     * @param offset for pagination, can be null
-     * @param orderCriteria a FileMetadatasOrderCriteria to order the results
-     * @return a FileMetadata list of the specified DatasetVersion
-     */
-    public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileMetadatasOrderCriteria orderCriteria) {
-        TypedQuery<FileMetadata> query = em.createQuery(getQueryStringFromFileMetadatasOrderCriteria(orderCriteria), FileMetadata.class)
-                .setParameter("datasetVersionId", datasetVersion.getId());
-        
-        if (limit == null && offset == null) {
-            query = query.setHint("eclipselink.left-join-fetch", "fm.dataFile.ingestRequest")
-                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.thumbnailForDataset")
-                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.dataTables")
-                    .setHint("eclipselink.left-join-fetch", "fm.fileCategories")
-                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.embargo")
-                    .setHint("eclipselink.left-join-fetch", "fm.datasetVersion")
-                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.releaseUser")
-                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.dataFileTags")
-                    .setHint("eclipselink.left-join-fetch", "fm.dataFile.creator");
-        } else {
-            // @todo: is there really no way to use offset-limit with left join hints?
-            if (limit != null) {
-                query = query.setMaxResults(limit);
-            }
-            if (offset != null) {
-                query = query.setFirstResult(offset);
-            }
-        }
-        return query.getResultList();
-    }
-
-    private String getQueryStringFromFileMetadatasOrderCriteria(FileMetadatasOrderCriteria orderCriteria) {
-        String queryString;
-        switch (orderCriteria) {
-            case NameZA:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL + " DESC";
-                break;
-            case Newest:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE + " DESC";
-                break;
-            case Oldest:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_DATE;
-                break;
-            case Size:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_SIZE;
-                break;
-            case Type:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_TYPE;
-                break;
-            default:
-                queryString = QUERY_STR_FIND_ALL_FILE_METADATAS_ORDER_BY_LABEL;
-                break;
-        }
-        return queryString;
-    }
 } // end class

From 381ddf59088808a536d58498e60514e1ea8557b8 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 11 Oct 2023 15:22:52 -0400
Subject: [PATCH 551/922] more commented-out code that needed to be removed
 before finalizing the pr. (#9763)

---
 .../edu/harvard/iq/dataverse/Dataset.java     | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 692a2ba0245..245bdf0efd2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -693,31 +693,12 @@ public Timestamp getCitationDate() {
         Timestamp citationDate = null;
         //Only calculate if this dataset doesn't use an alternate date field for publication date
         if (citationDateDatasetFieldType == null) {
-            // @todo: remove this commented-out code once/if the PR passes review - L.A. 
-            //List<DatasetVersion> versions = this.versions;
-            // TODo - is this ever not version 1.0 (or draft if not published yet)
-            //DatasetVersion oldest = versions.get(versions.size() - 1);
-            // - I believe the answer is yes, the oldest versions will always be 
-            // either 1.0 or draft - L.A. 
             citationDate = super.getPublicationDate();
             if (embargoCitationDate != null) {
                 if (citationDate.compareTo(embargoCitationDate) < 0) {
                     return embargoCitationDate;
                 }
             }
-            // @todo: remove this commented-out code once/if the PR passes review - L.A. 
-            /*if (oldest.isPublished()) {
-                List<FileMetadata> fms = oldest.getFileMetadatas();
-                for (FileMetadata fm : fms) {
-                    Embargo embargo = fm.getDataFile().getEmbargo();
-                    if (embargo != null) {
-                        Timestamp embDate = Timestamp.valueOf(embargo.getDateAvailable().atStartOfDay());
-                        if (citationDate.compareTo(embDate) < 0) {
-                            citationDate = embDate;
-                        }
-                    }
-                }
-            }*/
         }
         return citationDate;
     }

From 4c67f2a636699d51589fa815511ce4e1b3dc9d1f Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 6 Oct 2023 12:13:19 -0400
Subject: [PATCH 552/922] remove inefficient bucket check

---
 .../iq/dataverse/dataaccess/S3AccessIO.java       | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index 822ada0b83e..22216ee5c2b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -108,14 +108,13 @@ public S3AccessIO(T dvObject, DataAccessRequest req, String driverId) {
             if(!StringUtil.isEmpty(proxy)&&StringUtil.isEmpty(endpoint)) {
                 logger.severe(driverId + " config error: Must specify a custom-endpoint-url if proxy-url is specified");
             }
-            //Not sure this is needed but moving it from the open method for now since it definitely doesn't need to run every time an object is opened.
-            try {
-                if (bucketName == null || !s3.doesBucketExistV2(bucketName)) {
-                    throw new IOException("ERROR: S3AccessIO - You must create and configure a bucket before creating datasets.");
-                }
-            } catch (SdkClientException sce) {
-                throw new IOException("ERROR: S3AccessIO - Failed to look up bucket "+bucketName+" (is AWS properly configured?): " + sce.getMessage());
-            }
+
+            // FWIW: There used to be a check here to see if the bucket exists.
+            // It was very redundant (checking every time we access any file) and didn't do
+            // much but potentially make the failure (in the unlikely case a bucket doesn't
+            // exist/just disappeared) happen slightly earlier (here versus at the first
+            // file/metadata access).
+                    
         } catch (Exception e) {
             throw new AmazonClientException(
                         "Cannot instantiate a S3 client; check your AWS credentials and region",

From 90dfa42c9090ce9e4cf9dab1e8ed57776137a077 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 11 Oct 2023 16:41:38 -0400
Subject: [PATCH 553/922] Redesigned provider mechanism

---
 .../iq/dataverse/dataaccess/S3AccessIO.java   | 71 ++++++++++++++-----
 1 file changed, 52 insertions(+), 19 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index 22216ee5c2b..ee04bbcb853 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -4,6 +4,7 @@
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.HttpMethod;
 import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSCredentialsProviderChain;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
@@ -57,9 +58,11 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Optional;
 import java.util.Random;
 import java.util.function.Predicate;
 import java.util.logging.Logger;
@@ -1180,29 +1183,59 @@ private static AmazonS3 getClient(String driverId) {
             // Boolean is inverted, otherwise setting dataverse.files.<id>.chunked-encoding=false would result in leaving Chunked Encoding enabled
             s3CB.setChunkedEncodingDisabled(!s3chunkedEncoding);
 
-            /**
-             * Pass in a string value if this storage driver should use a non-default AWS S3 profile.
-             * The default is "default" which should work when only one profile exists.
+            /** Configure credentials for the S3 client. There are multiple mechanisms available. 
+             * Role-based/instance credentials are globally defined while the other mechanisms (profile, static)
+             * are defined per store. The logic below assures that 
+             * * if a store specific profile or static credentials are explicitly set, they will be used in preference to the global role-based credentials. 
+             * * if a store specific role-based credentials are explicitly set, they will be used in preference to the global instance credentials,
+             * * if a profile and static credentials are both explicitly set, the profile will be used preferentially, and 
+             * * if no store-specific credentials are set, the global credentials will be preferred over using any "default" profile credentials that are found.
              */
-            String s3profile = System.getProperty("dataverse.files." + driverId + ".profile","default");
-            ProfileCredentialsProvider profileCredentials = new ProfileCredentialsProvider(s3profile);
-    
-            // Try to retrieve credentials via Microprofile Config API, too. For production use, you should not use env
-            // vars or system properties to provide these, but use the secrets config source provided by Payara.
-            AWSStaticCredentialsProvider staticCredentials = new AWSStaticCredentialsProvider(
-                new BasicAWSCredentials(
-                    config.getOptionalValue("dataverse.files." + driverId + ".access-key", String.class).orElse(""),
-                    config.getOptionalValue("dataverse.files." + driverId + ".secret-key", String.class).orElse("")
-                ));
-            
-            //Add role-based provider as in the default provider chain
-            InstanceProfileCredentialsProvider instanceCredentials = InstanceProfileCredentialsProvider.getInstance();
+            ArrayList<AWSCredentialsProvider> providers = new ArrayList<>();
+
+            String s3profile = System.getProperty("dataverse.files." + driverId + ".profile");
+            boolean allowInstanceCredentials = true;
+            // Assume that instance credentials should not be used if the profile is
+            // actually set for this store or if static creds are provided (below).
+            if (s3profile != null) {
+                allowInstanceCredentials = false;
+            }
+            // Try to retrieve credentials via Microprofile Config API, too. For production
+            // use, you should not use env vars or system properties to provide these, but 
+            // use the secrets config source provided by Payara.
+            Optional<String> accessKey = config.getOptionalValue("dataverse.files." + driverId + ".access-key", String.class);
+            Optional<String> secretKey = config.getOptionalValue("dataverse.files." + driverId + ".secret-key", String.class);
+            if (accessKey.isPresent() && secretKey.isPresent()) {
+                allowInstanceCredentials = false;
+                AWSStaticCredentialsProvider staticCredentials = new AWSStaticCredentialsProvider(
+                        new BasicAWSCredentials(
+                                accessKey.orElse(""),
+                                secretKey.orElse("")));
+                providers.add(staticCredentials);
+            } else if (s3profile == null) {
+                //Only use the default profile when it isn't explicitly set for this store when there are no static creds (otherwise it will be preferred).
+                s3profile = "default";
+            }
+            if (s3profile != null) {
+                ProfileCredentialsProvider profileCredentials = new ProfileCredentialsProvider(s3profile);
+                providers.add(profileCredentials);
+            }
+
+            if (allowInstanceCredentials) {
+                // Add role-based provider as in the default provider chain
+                InstanceProfileCredentialsProvider instanceCredentials = InstanceProfileCredentialsProvider.getInstance();
+                providers.add(instanceCredentials);
+            }
             // Add all providers to chain - the first working provider will be used
-            // (role-based is first in the default cred provider chain, so we're just
+            // (role-based is first in the default cred provider chain (if no profile or
+            // static creds are explicitly set for the store), so we're just
             // reproducing that, then profile, then static credentials as the fallback)
-            AWSCredentialsProviderChain providerChain = new AWSCredentialsProviderChain(instanceCredentials, profileCredentials, staticCredentials);
+
+            // As the order is the reverse of how we added providers, we reverse the list here
+            Collections.reverse(providers);
+            AWSCredentialsProviderChain providerChain = new AWSCredentialsProviderChain(providers);
             s3CB.setCredentials(providerChain);
-            
+
             // let's build the client :-)
             AmazonS3 client =  s3CB.build();
             driverClientMap.put(driverId,  client);

From dcca52566958fba3f58698766f9696723fcebfc0 Mon Sep 17 00:00:00 2001
From: qqmyers <jim.myers@computer.org>
Date: Thu, 12 Oct 2023 09:28:42 -0400
Subject: [PATCH 554/922] Good cleanup

Co-authored-by: Oliver Bertuch <poikilotherm@users.noreply.github.com>
---
 .../harvard/iq/dataverse/dataaccess/S3AccessIO.java    | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index ee04bbcb853..a66686ac648 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -1209,22 +1209,20 @@ private static AmazonS3 getClient(String driverId) {
                 allowInstanceCredentials = false;
                 AWSStaticCredentialsProvider staticCredentials = new AWSStaticCredentialsProvider(
                         new BasicAWSCredentials(
-                                accessKey.orElse(""),
-                                secretKey.orElse("")));
+                                accessKey.get(),
+                                secretKey.get()));
                 providers.add(staticCredentials);
             } else if (s3profile == null) {
                 //Only use the default profile when it isn't explicitly set for this store when there are no static creds (otherwise it will be preferred).
                 s3profile = "default";
             }
             if (s3profile != null) {
-                ProfileCredentialsProvider profileCredentials = new ProfileCredentialsProvider(s3profile);
-                providers.add(profileCredentials);
+                providers.add(new ProfileCredentialsProvider(s3profile));
             }
 
             if (allowInstanceCredentials) {
                 // Add role-based provider as in the default provider chain
-                InstanceProfileCredentialsProvider instanceCredentials = InstanceProfileCredentialsProvider.getInstance();
-                providers.add(instanceCredentials);
+                providers.add(InstanceProfileCredentialsProvider.getInstance());
             }
             // Add all providers to chain - the first working provider will be used
             // (role-based is first in the default cred provider chain (if no profile or

From ada8cc7a713c8074378c7732d4cf30688d50f9cf Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 13 Oct 2023 10:44:14 +0100
Subject: [PATCH 555/922] Fixed: curl examples in docs for deaccession dataset

---
 doc/sphinx-guides/source/api/native-api.rst | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index e51ca0055b6..1dc1ab13d9f 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -1383,21 +1383,31 @@ Deaccession Dataset
 
 Given a version of a dataset, updates its status to deaccessioned.
 
+The JSON body required to deaccession a dataset (``deaccession.json``) looks like this::
+
+  {
+    "deaccessionReason": "Description of the deaccession reason.",
+    "deaccessionForwardURL": "https://demo.dataverse.org"
+  }
+
+
+Note that the field ``deaccessionForwardURL`` is optional.
+
 .. code-block:: bash
 
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export ID=24
   export VERSIONID=1.0
-  export JSON='{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
+  export FILE_PATH=deaccession.json
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession" -d "$JSON"
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession" -H "Content-type:application/json" --upload-file $FILE_PATH
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -d '{"deaccessionReason":"Description of the deaccession reason.", "deaccessionForwardURL":"https://demo.dataverse.org"}'
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -H "Content-type:application/json" --upload-file deaccession.json
 
 .. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be published since it is already deaccessioned.
 

From 1f0efddbd6cb4e10b7f5924dbd338105f18add81 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 13 Oct 2023 11:35:44 +0100
Subject: [PATCH 556/922] Fixed: permission checks in
 GetSpecificPublishedDatasetVersionCommand

---
 ...etSpecificPublishedDatasetVersionCommand.java |  3 ++-
 .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 16 ++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java     | 14 +++++++++-----
 3 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java
index 879a694ef57..a87eb8a99a5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetSpecificPublishedDatasetVersionCommand.java
@@ -8,6 +8,7 @@
 
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
@@ -40,7 +41,7 @@ public GetSpecificPublishedDatasetVersionCommand(DataverseRequest aRequest, Data
     @Override
     public DatasetVersion execute(CommandContext ctxt) throws CommandException {
         for (DatasetVersion dsv : ds.getVersions()) {
-            if (dsv.isReleased() || (includeDeaccessioned && dsv.isDeaccessioned())) {
+            if (dsv.isReleased() || (includeDeaccessioned && dsv.isDeaccessioned() && ctxt.permissions().requestOn(getRequest(), ds).has(Permission.EditDataset))) {
                 if (dsv.getVersionNumber().equals(majorVersion) && dsv.getMinorVersionNumber().equals(minorVersion)) {
                     return dsv;
                 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 2d52a6c6e15..ee81d3f67f4 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3543,6 +3543,14 @@ public void getVersionFiles() throws IOException, InterruptedException {
 
         fileMetadatasCount = getVersionFilesResponseTabularTagName.jsonPath().getList("data").size();
         assertEquals(1, fileMetadatasCount);
+
+        // Test that the dataset files for a deaccessioned dataset cannot be accessed by a guest
+        // By latest published version
+        Response getDatasetVersionResponse = UtilIT.getVersionFiles(datasetId, DS_VERSION_LATEST_PUBLISHED, null, null, null, null, null, null, null, null, true, null);
+        getDatasetVersionResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+        // By specific version 1.0
+        getDatasetVersionResponse = UtilIT.getVersionFiles(datasetId, "1.0", null, null, null, null, null, null, null, null, true, null);
+        getDatasetVersionResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
     }
 
     @Test
@@ -3620,6 +3628,14 @@ public void getVersionFileCounts() throws IOException {
 
         responseJsonPath = getVersionFileCountsResponseDeaccessioned.jsonPath();
         assertEquals(4, (Integer) responseJsonPath.get("data.total"));
+
+        // Test that the dataset file counts for a deaccessioned dataset cannot be accessed by a guest
+        // By latest published version
+        Response getDatasetVersionResponse = UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST_PUBLISHED, true, null);
+        getDatasetVersionResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+        // By specific version 1.0
+        getDatasetVersionResponse = UtilIT.getVersionFileCounts(datasetId, "1.0", true, null);
+        getDatasetVersionResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
     }
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 93a7cc64082..434dc6d26f1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3290,9 +3290,11 @@ static Response getVersionFiles(Integer datasetId,
                                     boolean includeDeaccessioned,
                                     String apiToken) {
         RequestSpecification requestSpecification = given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json")
                 .queryParam("includeDeaccessioned", includeDeaccessioned);
+        if (apiToken != null) {
+            requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken);
+        }
         if (limit != null) {
             requestSpecification = requestSpecification.queryParam("limit", limit);
         }
@@ -3372,10 +3374,12 @@ static Response createFileEmbargo(Integer datasetId, Integer fileId, String date
     }
 
     static Response getVersionFileCounts(Integer datasetId, String version, boolean includeDeaccessioned, String apiToken) {
-        return given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .queryParam("includeDeaccessioned", includeDeaccessioned)
-                .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
+        RequestSpecification requestSpecification = given()
+                .queryParam("includeDeaccessioned", includeDeaccessioned);
+        if (apiToken != null) {
+            requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken);
+        }
+        return requestSpecification.get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts");
     }
 
     static Response setFileCategories(String dataFileId, String apiToken, List<String> categories) {

From 4b5ad8fac1c1733c73ad0e2f5d7e1e47155895bc Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 13 Oct 2023 10:04:25 -0400
Subject: [PATCH 557/922] rename sql script #9763

avoid conflict with V6.0.0.1__9599-guestbook-at-request.sql
---
 ...rgocitationdate.sql => V6.0.0.2__9763-embargocitationdate.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V6.0.0.1__9763-embargocitationdate.sql => V6.0.0.2__9763-embargocitationdate.sql} (100%)

diff --git a/src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql b/src/main/resources/db/migration/V6.0.0.2__9763-embargocitationdate.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql
rename to src/main/resources/db/migration/V6.0.0.2__9763-embargocitationdate.sql

From 12ba35e9b9c4f0396ed942ea30a832e6a57c22c9 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 13 Oct 2023 17:14:17 +0100
Subject: [PATCH 558/922] Fixed: failing tests after develop merge

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 6626b18219c..34eccd3172a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3622,8 +3622,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         assertEquals(1, responseCountPerAccessStatusMap.get(FileSearchCriteria.FileAccessStatus.EmbargoedThenPublic.toString()));
 
         // Test content type criteria
-        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST_PUBLISHED, "image/png", null, null, null, null, false, apiToken);
-
+        getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, DS_VERSION_LATEST, "image/png", null, null, null, null, false, apiToken);
         getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         responseJsonPath = getVersionFileCountsResponse.jsonPath();
@@ -3760,7 +3759,7 @@ public void getVersionFileCounts() throws IOException, InterruptedException {
         getVersionFileCountsResponseDeaccessioned.then().assertThat().statusCode(OK.getStatusCode());
 
         responseJsonPath = getVersionFileCountsResponseDeaccessioned.jsonPath();
-        assertEquals(4, (Integer) responseJsonPath.get("data.total"));
+        assertEquals(5, (Integer) responseJsonPath.get("data.total"));
 
         // Test that the dataset file counts for a deaccessioned dataset cannot be accessed by a guest
         // By latest published version

From beed44473f5a51d2a9d69fd31353c4900f8391ae Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 13 Oct 2023 13:44:23 -0400
Subject: [PATCH 559/922] re-apply #9892

---
 src/main/webapp/guestbook-terms-popup-fragment.xhtml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 69cc9fae55c..34df0c79390 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -7,7 +7,8 @@
                 xmlns:o="http://omnifaces.org/ui"
                 xmlns:jsf="http://xmlns.jcp.org/jsf"
                 xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs">
-
+                
+    <ui:fragment rendered="#{DatasetPage.editMode != 'CREATE'}">
         <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
         <p:focus context="guestbookUIFragment"/>
         <ui:fragment rendered="#{termsGuestbookPopupAction != bundle.download}">
@@ -321,4 +322,5 @@
 
             <button type="button" class="btn btn-default" onclick="PF('guestbookAndTermsPopup').hide();PF('blockDatasetForm').hide();" value="#{bundle.cancel}">#{bundle.cancel}</button>
         </div>
+    </ui:fragment>
 </ui:composition>

From 53b73e09f28751c3e17150e17908df5fa3f308b4 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 13 Oct 2023 14:07:37 -0400
Subject: [PATCH 560/922] remove file with "TODO - remove!"

---
 .../webapp/file-download-popup-fragment.xhtml | 305 ------------------
 1 file changed, 305 deletions(-)
 delete mode 100644 src/main/webapp/file-download-popup-fragment.xhtml

diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml
deleted file mode 100644
index 3a64ca4a3a2..00000000000
--- a/src/main/webapp/file-download-popup-fragment.xhtml
+++ /dev/null
@@ -1,305 +0,0 @@
-<!--  TODO - remove! -->
-<ui:composition xmlns="http://www.w3.org/1999/xhtml"
-                xmlns:h="http://java.sun.com/jsf/html"
-                xmlns:f="http://java.sun.com/jsf/core"
-                xmlns:ui="http://java.sun.com/jsf/facelets"
-                xmlns:c="http://java.sun.com/jsp/jstl/core"
-                xmlns:p="http://primefaces.org/ui"
-                xmlns:o="http://omnifaces.org/ui"
-                xmlns:jsf="http://xmlns.jcp.org/jsf"
-                xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs">
-
-    <ui:fragment rendered="#{DatasetPage.editMode != 'CREATE'}">
-
-        <o:importFunctions type="edu.harvard.iq.dataverse.util.MarkupChecker" />
-        <p:focus context="guestbookUIFragment"/>
-        <p class="help-block">
-            #{bundle['file.downloadDialog.tip']}
-        </p>
-        <p:fragment id="guestbookMessages">
-            <div class="container messagePanel">
-                <iqbs:messages collapsible="true" />
-            </div>
-        </p:fragment>
-        
-        <div class="form-horizontal terms">
-        
-        <div class="form-group">
-            <label jsf:for="fdLicense" class="col-sm-3 control-label">
-                #{bundle['file.dataFilesTab.terms.list.license']} </label>
-            <div class="col-sm-9">
-                <p class="help-block">
-                    <h:outputText
-                        value="#{bundle['file.dataFilesTab.terms.list.license.view.description']}"
-                        escape="false" />
-                </p>
-                <p>
-                    <o:importFunctions
-                        type="edu.harvard.iq.dataverse.dataset.DatasetUtil" />
-                    <img
-                        src="#{DatasetUtil:getLicenseIcon(workingVersion)}"
-                        title="#{DatasetUtil:getLicenseDescription(workingVersion)}"
-                        style="display: none" onload="this.style.display='inline'" />
-                    <a 
-                        href="#{DatasetUtil:getLicenseURI(workingVersion)}"
-                        title="#{DatasetUtil:getLicenseDescription(workingVersion)}"
-                        target="_blank">#{DatasetUtil:getLicenseName(workingVersion)}
-                    </a>
-                    <ui:fragment
-                        rendered="#{empty workingVersion.termsOfUseAndAccess.license}">
-                        <h:outputText
-                            value="#{bundle['file.dataFilesTab.terms.list.license.customterms.txt']}"
-                            escape="false" />
-                    </ui:fragment>
-                </p>
-            </div>
-        </div>
-        
-
-        <ui:fragment rendered="#{empty workingVersion.termsOfUseAndAccess.license}">
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.termsOfUse}">
-                <label jsf:for="fdTermsOfuse" class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.termsOfUse']} 
-                </label>
-                <h:outputText id="fdTermsOfUse" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfUse)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.confidentialityDeclaration}">
-                <label jsf:for="fdConfidentialityDeclaration"
-                    class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.declaration']}
-                </label>
-                <h:outputText id="fdConfidentialityDeclaration" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.confidentialityDeclaration)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.specialPermissions}">
-                <label jsf:for="fdSpecialPermissions" class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.permissions']}
-                </label>
-                <h:outputText id="fdSpecialPermissions" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.specialPermissions)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.restrictions}">
-                <label jsf:for="fdRestrictions" class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.restrictions']}
-                </label>
-                <h:outputText id="fdRestrictions" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.restrictions)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.citationRequirements}">
-                <label jsf:for="fdCitationRequirements"
-                    class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.citationRequirements']}
-                </label>
-                <h:outputText id="fdCitationRequirements" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.citationRequirements)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.depositorRequirements}">
-                <label jsf:for="fdDepositorRequirements"
-                    class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.depositorRequirements']}
-                </label>
-                <h:outputText id="fdDepositorRequirements" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.depositorRequirements)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.conditions}">
-                <label jsf:for="fdConditions" class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.conditions']}
-                </label>
-                <h:outputText id="fdConditions" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.conditions)}"
-                    escape="false" />
-            </div>
-            <div class="form-group"
-                jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.disclaimer}">
-                <label jsf:for="fdDisclaimer" class="col-sm-3 control-label">
-                    #{bundle['file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer']}
-                </label>
-                <h:outputText id="fdDisclaimer" class="col-sm-6"
-                    value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.disclaimer)}"
-                    escape="false" />
-            </div>
-        </ui:fragment>
-        <div class="form-group"
-            jsf:rendered="#{!empty workingVersion.termsOfUseAndAccess.termsOfAccess and hasRestrictedFile}">
-            <label jsf:for="fdTermsOfAccess" class="col-sm-3 control-label">
-                #{bundle['file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess']}
-            </label>
-            <h:outputText id="fdTermsOfAccess" class="col-sm-6"
-                value="#{MarkupChecker:sanitizeBasicHTML(workingVersion.termsOfUseAndAccess.termsOfAccess)}"
-                escape="false" />
-        </div>
-        <p:fragment rendered="#{workingVersion.dataset.guestbook != null and workingVersion.dataset.guestbook.enabled and guestbookAndTermsPopupRequired}" id="guestbookUIFragment">
-                <!--
-                Only validate the active context. For example, the user might be filling out
-                the guestbook on the Preview tab rather than after clicking the "download file"
-                button, even though the form is the same and on the same page. Without this boolean
-                in place, invalid fields on the other form (the non-active form) are checked, leading
-                to a validation failure.
-                -->
-                <ui:param name="validateThisContext" value="DO_GB_VALIDATION_#{popupContext}"/>
-                <div class="form-group">
-                    <label class="col-sm-3 control-label" for="guestbookuser_nameText">
-                            #{bundle.name}
-                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.nameRequired}" />
-                    </label>
-                    <div class="col-sm-6">
-                        <p:inputText id="guestbookuser_nameText"
-                                     required="#{param[validateThisContext] and workingVersion.dataset.guestbook.nameRequired}"
-                                     styleClass="form-control" value="#{guestbookResponse.name}"
-                                     requiredMessage="#{bundle['requiredField']}"
-                                     >
-                        </p:inputText>
-                        <p:message id="nameMessages" for="guestbookuser_nameText" display="text"/>
-                    </div>
-                </div>
-                <div class="form-group">
-                    <label class="col-sm-3 control-label" for="guestbookuser_email">
-                        #{bundle.email}
-                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.emailRequired}" />
-                    </label>
-                    <div class="col-sm-6">
-                        <p:inputText id="guestbookuser_email" required="#{param[validateThisContext] and workingVersion.dataset.guestbook.emailRequired}"
-                                     styleClass="form-control" value="#{guestbookResponse.email}"
-                                     requiredMessage="#{bundle['requiredField']}"
-                                     >
-                        </p:inputText>  
-                        <p:message id="emailMessages" for="guestbookuser_email" display="text"/>
-                    </div>
-                </div>
-                <div class="form-group">
-                    <label class="col-sm-3 control-label" for="guestbookuser_institution">
-                        #{bundle.institution}
-                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.institutionRequired}" />
-                    </label>
-                    <div class="col-sm-6">
-                        <p:inputText id="guestbookuser_institution" required="#{param[validateThisContext] and workingVersion.dataset.guestbook.institutionRequired}"
-                                     styleClass="form-control" value="#{guestbookResponse.institution}"
-                                     requiredMessage="#{bundle['requiredField']}"
-                                     >
-                        </p:inputText>
-                        <p:message id="institutionMessages" for="guestbookuser_institution" display="text"/>
-                    </div>
-                </div>
-                <div class="form-group">
-                    <label class="col-sm-3 control-label" for="guestbookuser_position">
-                        #{bundle.position}
-                        <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{workingVersion.dataset.guestbook.positionRequired}" />
-                    </label>
-                    <div class="col-sm-6">
-                        <p:inputText id="guestbookuser_position"  required="#{param[validateThisContext] and workingVersion.dataset.guestbook.positionRequired}"
-                                     styleClass="form-control" value="#{guestbookResponse.position}"
-                                     requiredMessage="#{bundle['requiredField']}"
-                                     >
-                        </p:inputText>
-                        <p:message id="positionMessages" for="guestbookuser_position" display="text"/>
-                    </div>
-                </div>
-                <div class="form-group" jsf:rendered="#{!empty workingVersion.dataset.guestbook.customQuestions}">
-                    <label class="col-sm-3 control-label">
-                        #{bundle['dataset.guestbookResponse.guestbook.additionalQuestions']}
-                    </label>
-                    <div class="col-sm-6">
-                        <ui:repeat value="#{guestbookResponse.customQuestionResponses}" var="customQuestionResponse">
-                            <div class="text-left">
-                                <label class="control-label">
-                                    <h:outputText value="#{customQuestionResponse.customQuestion.questionString} "/>
-                                    <span class="glyphicon glyphicon-asterisk text-danger" jsf:rendered="#{customQuestionResponse.customQuestion.required}" />
-                                </label>
-                                <p:inputText id="customQuestionResponse"
-                                             styleClass="form-control" value="#{customQuestionResponse.response}"
-                                             required="#{param[validateThisContext] and customQuestionResponse.customQuestion.required}"
-                                             rendered="#{customQuestionResponse.customQuestion.questionType=='text'}"
-                                             requiredMessage="#{bundle['requiredField']}">
-                                </p:inputText>
-                                <p:message id="cqMessages" for="customQuestionResponse" display="text"/>
-                                <p:inputTextarea id="customQuestionResponseTextArea" rows="8" autoResize="true"
-                                             styleClass="form-control"
-                                             value="#{customQuestionResponse.response}"
-                                             required="#{param['DO_GB_VALIDATION'] and customQuestionResponse.customQuestion.required}"
-                                             rendered="#{customQuestionResponse.customQuestion.questionType=='textarea'}"
-                                             requiredMessage="#{bundle['requiredField']}"/>
-                                <p:message id="cqMessagesTA" for="customQuestionResponseTextArea" display="text"/>
-                                <p:selectOneMenu id="customQuestionResponseSelect"
-                                                 styleClass="form-control" value="#{customQuestionResponse.response}"
-                                                 required="#{param[validateThisContext] and customQuestionResponse.customQuestion.required}"
-                                                 rendered="#{customQuestionResponse.customQuestion.questionType=='options'}"
-                                                 requiredMessage="#{bundle['requiredField']}">
-                                    <f:selectItem itemLabel="#{bundle.select}" itemValue="" noSelectionOption="true" />
-                                    <f:selectItems value="#{customQuestionResponse.responseSelectItems}" />
-                                </p:selectOneMenu>
-                                <p:message id="cqrsMessages" for="customQuestionResponseSelect" display="text"/>
-                            </div>
-                        </ui:repeat>
-                    </div>
-                </div>
-            </p:fragment>
-        </div>
-        <div class="button-block">
-            <!--
-            The "process" directive below is very important. Without it, the
-            setters on the GuestbookResponse object can be called twice leading
-            to form values (name, email, etc) to be overwritten by the object in
-            the other context. For example, "name" in the Preview tab could get
-            overwritten by "name" in the download popup with either a blank
-            value or a prefilled value, leading to a botched guestbook entry.
-            
-            Experimentation was done with adding process="downloadPopup" to the
-            non-Preview tab buttons but this caused a ComponentNotFoundException
-            and didn't solve any problems. If you add logging to print out
-            setName from Guestbook response, you can see that the setters are
-            called on the GuestbookResponse object from the Preview tab but they
-            are later overwritten by the setters on the GuestbookResponse object
-            from the Download popup.
-            -->
-            <!--REGULAR DOWNLOAD BUTTON, NO EXTERNAL TOOL, NOT THE PREVIEW TAB-->
-            <!--Note: the guestbookResponse.fileFormat is being set in xhtml via the initial download buttons in file-download-button-fragment.xhtml -->
-            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
-                             rendered="#{guestbookResponse.fileFormat != 'externalTool' and
-                                         guestbookResponse.fileFormat != 'package' and
-                                         popupContext != 'previewTab'}"
-                             actionListener="#{fileDownloadHelper.writeGuestbookAndStartDownload(guestbookResponse)}"
-                             update="guestbookUIFragment">
-                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
-            </p:commandButton>
-            <!--PREVIEW TAB BUTTON-->
-            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
-                             rendered="#{popupContext == 'previewTab'}"
-                             actionListener="#{FilePage.showPreview(guestbookResponse)}"
-                             process="previewTab"
-                             update="fileForm:tabView">
-                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
-            </p:commandButton>
-            <!--EXTERNAL TOOL BUTTON-->
-            <!--On the dataset page (but not the file page), "tool" is null so we get the tool from the guestbookResponse.-->
-            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" rendered="#{guestbookResponse.fileFormat == 'externalTool'}"
-                             action="#{fileDownloadHelper.writeGuestbookAndLaunchExploreTool(guestbookResponse, fileMetadata, tool)}"
-                             update="guestbookUIFragment">
-                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
-            </p:commandButton>
-            <!--PACKAGE FILE BUTTON-->
-            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}" rendered="#{guestbookResponse.fileFormat == 'package'}"
-                             actionListener="#{fileDownloadHelper.writeGuestbookAndLaunchPackagePopup(guestbookResponse)}"
-                             update="guestbookUIFragment">
-                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
-            </p:commandButton>
-            <!--CANCEL BUTTON We don't render the Cancel button on the Preview tab because there is no popup to close.-->
-            <button class="btn btn-link" onclick="PF('downloadPopup').hide();PF('blockDatasetForm').hide();" type="button" jsf:rendered="#{popupContext != 'previewTab'}">
-                #{bundle.cancel}
-            </button>
-        </div>
-    </ui:fragment>
-</ui:composition>
\ No newline at end of file

From f47867ee34e93e14efaca2fba414e202d234c1c6 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 15 Oct 2023 19:24:09 -0400
Subject: [PATCH 561/922] renaming the flyway script since 6.0.0.1 has already
 been merged. (#9763)

---
 ...rgocitationdate.sql => V6.0.0.2__9763-embargocitationdate.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V6.0.0.1__9763-embargocitationdate.sql => V6.0.0.2__9763-embargocitationdate.sql} (100%)

diff --git a/src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql b/src/main/resources/db/migration/V6.0.0.2__9763-embargocitationdate.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.0.0.1__9763-embargocitationdate.sql
rename to src/main/resources/db/migration/V6.0.0.2__9763-embargocitationdate.sql

From 35f69517ea2139c2e742b7d7b28e1b88dcdd9ef5 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 15 Oct 2023 20:22:09 -0400
Subject: [PATCH 562/922] Switching to the new version of gdcc/xoai, v5.2.0
 (#9910)

---
 modules/dataverse-parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 19f78415280..1d99c1cd3d8 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -165,7 +165,7 @@
         <apache.httpcomponents.core.version>4.4.14</apache.httpcomponents.core.version>
         
         <!-- NEW gdcc XOAI library implementation -->
-        <gdcc.xoai.version>5.1.0</gdcc.xoai.version>
+        <gdcc.xoai.version>5.2.0</gdcc.xoai.version>
     
         <!-- Testing dependencies -->
         <testcontainers.version>1.19.0</testcontainers.version>

From 4182b036f24ba8402ffe7f2c304ed4026fa7874d Mon Sep 17 00:00:00 2001
From: Abhinav Rana <142827270+AR-2910@users.noreply.github.com>
Date: Mon, 16 Oct 2023 07:50:09 +0530
Subject: [PATCH 563/922] Update config.rst

Adding link to "Dataverse General User Interface Translation Guide for Weblate" in the "Tools For Translators" section. Issue #9512.
---
 doc/sphinx-guides/source/installation/config.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 086b0a80895..ce8876b012c 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1276,6 +1276,8 @@ The list below depicts a set of tools that can be used to ease the amount of wor
 
 - `easyTranslationHelper <https://github.com/universidadeaveiro/easyTranslationHelper>`_, a tool developed by `University of Aveiro <https://www.ua.pt/>`_.
 
+- `Dataverse General User Interface Translation Guide for Weblate <https://doi.org/10.5281/zenodo.4807371>`_, a guide produced as part of the `SSHOC Dataverse Translation <https://www.sshopencloud.eu/news/workshop-notes-sshoc-dataverse-translation-follow-event/>`_ event.
+
 .. _Web-Analytics-Code:
 
 Web Analytics Code

From cea36d6aadcc3a21d70c3029b498a279256d6c07 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 16 Oct 2023 12:19:15 +0100
Subject: [PATCH 564/922] Added: file search criteria to getDownloadSize
 datasets API endpoint

---
 .../DatasetVersionFilesServiceBean.java       | 23 +++++++++++--------
 .../harvard/iq/dataverse/api/Datasets.java    | 19 ++++++++++++++-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 21 +++++++++++------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 22 +++++++++++++++---
 4 files changed, 64 insertions(+), 21 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 2c14498caa9..6ea9262bbc4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -53,7 +53,7 @@ public enum FileOrderCriteria {
     }
 
     /**
-     * Mode to base the search in {@link DatasetVersionFilesServiceBean#getFilesDownloadSize(DatasetVersion, FileDownloadSizeMode)}
+     * Mode to base the search in {@link DatasetVersionFilesServiceBean#getFilesDownloadSize(DatasetVersion, FileSearchCriteria, FileDownloadSizeMode)}
      * <p>
      * All: Includes both archival and original sizes for tabular files
      * Archival: Includes only the archival size for tabular files
@@ -191,16 +191,17 @@ public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Intege
      * Returns the total download size of all files for a particular DatasetVersion
      *
      * @param datasetVersion the DatasetVersion to access
+     * @param searchCriteria for retrieving only files matching this criteria
      * @param mode           a FileDownloadSizeMode to base the search on
      * @return long value of total file download size
      */
-    public long getFilesDownloadSize(DatasetVersion datasetVersion, FileDownloadSizeMode mode) {
+    public long getFilesDownloadSize(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria, FileDownloadSizeMode mode) {
         return switch (mode) {
             case All ->
-                    Long.sum(getOriginalTabularFilesSize(datasetVersion), getArchivalFilesSize(datasetVersion, false));
+                    Long.sum(getOriginalTabularFilesSize(datasetVersion, searchCriteria), getArchivalFilesSize(datasetVersion, false, searchCriteria));
             case Original ->
-                    Long.sum(getOriginalTabularFilesSize(datasetVersion), getArchivalFilesSize(datasetVersion, true));
-            case Archival -> getArchivalFilesSize(datasetVersion, false);
+                    Long.sum(getOriginalTabularFilesSize(datasetVersion, searchCriteria), getArchivalFilesSize(datasetVersion, true, searchCriteria));
+            case Archival -> getArchivalFilesSize(datasetVersion, false, searchCriteria);
         };
     }
 
@@ -301,22 +302,24 @@ private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> qu
         }
     }
 
-    private long getOriginalTabularFilesSize(DatasetVersion datasetVersion) {
+    private long getOriginalTabularFilesSize(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        Long result = queryFactory
+        JPAQuery<?> baseQuery = queryFactory
                 .from(fileMetadata)
                 .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
                 .from(dataTable)
-                .where(dataTable.dataFile.eq(fileMetadata.dataFile))
-                .select(dataTable.originalFileSize.sum()).fetchFirst();
+                .where(dataTable.dataFile.eq(fileMetadata.dataFile));
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
+        Long result = baseQuery.select(dataTable.originalFileSize.sum()).fetchFirst();
         return (result == null) ? 0 : result;
     }
 
-    private long getArchivalFilesSize(DatasetVersion datasetVersion, boolean ignoreTabular) {
+    private long getArchivalFilesSize(DatasetVersion datasetVersion, boolean ignoreTabular, FileSearchCriteria searchCriteria) {
         JPAQueryFactory queryFactory = new JPAQueryFactory(em);
         JPAQuery<?> baseQuery = queryFactory
                 .from(fileMetadata)
                 .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
+        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
         Long result;
         if (ignoreTabular) {
             result = baseQuery.where(fileMetadata.dataFile.dataTables.isEmpty()).select(fileMetadata.dataFile.filesize.sum()).fetchFirst();
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index f8929c5e8d8..8605b4772f4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2992,11 +2992,28 @@ public Response getStorageSize(@Context ContainerRequestContext crc, @PathParam(
     public Response getDownloadSize(@Context ContainerRequestContext crc,
                                     @PathParam("identifier") String dvIdtf,
                                     @PathParam("versionId") String version,
+                                    @QueryParam("contentType") String contentType,
+                                    @QueryParam("accessStatus") String accessStatus,
+                                    @QueryParam("categoryName") String categoryName,
+                                    @QueryParam("tabularTagName") String tabularTagName,
+                                    @QueryParam("searchText") String searchText,
                                     @QueryParam("mode") String mode,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
 
         return response(req -> {
+            FileSearchCriteria fileSearchCriteria;
+            try {
+                fileSearchCriteria = new FileSearchCriteria(
+                        contentType,
+                        accessStatus != null ? FileSearchCriteria.FileAccessStatus.valueOf(accessStatus) : null,
+                        categoryName,
+                        tabularTagName,
+                        searchText
+                );
+            } catch (IllegalArgumentException e) {
+                return badRequest(BundleUtil.getStringFromBundle("datasets.api.version.files.invalid.access.status", List.of(accessStatus)));
+            }
             DatasetVersionFilesServiceBean.FileDownloadSizeMode fileDownloadSizeMode;
             try {
                 fileDownloadSizeMode = mode != null ? DatasetVersionFilesServiceBean.FileDownloadSizeMode.valueOf(mode) : DatasetVersionFilesServiceBean.FileDownloadSizeMode.All;
@@ -3004,7 +3021,7 @@ public Response getDownloadSize(@Context ContainerRequestContext crc,
                 return error(Response.Status.BAD_REQUEST, "Invalid mode: " + mode);
             }
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers);
-            long datasetStorageSize = datasetVersionFilesServiceBean.getFilesDownloadSize(datasetVersion, fileDownloadSizeMode);
+            long datasetStorageSize = datasetVersionFilesServiceBean.getFilesDownloadSize(datasetVersion, fileSearchCriteria, fileDownloadSizeMode);
             String message = MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.download"), datasetStorageSize);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
             jsonObjectBuilder.add("message", message);
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 34eccd3172a..66a67887405 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3863,7 +3863,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         int expectedTextFilesStorageSize = testFileSize1 + testFileSize2;
 
         // Get the total size when there are no tabular files
-        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedTextFilesStorageSize));
 
@@ -3878,7 +3878,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         Thread.sleep(2000);
 
         // Get the total size ignoring the original tabular file sizes
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Archival.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Archival.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         int actualSizeIgnoringOriginalTabularSizes = Integer.parseInt(getDownloadSizeResponse.getBody().jsonPath().getString("data.storageSize"));
@@ -3889,7 +3889,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         // Get the total size including only original sizes and ignoring archival sizes for tabular files
         int expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedTextFilesStorageSize;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingOnlyOriginalForTabular));
 
@@ -3897,13 +3897,13 @@ public void getDownloadSize() throws IOException, InterruptedException {
         int tabularArchivalSize = actualSizeIgnoringOriginalTabularSizes - expectedTextFilesStorageSize;
         int expectedSizeIncludingAllSizes = tabularArchivalSize + tabularOriginalSize + expectedTextFilesStorageSize;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
 
         // Get the total size sending invalid file download size mode
         String invalidMode = "invalidMode";
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, invalidMode, apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, invalidMode, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid mode: " + invalidMode));
 
@@ -3917,15 +3917,22 @@ public void getDownloadSize() throws IOException, InterruptedException {
         // Get the total size including only original sizes and ignoring archival sizes for tabular files
         expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedSizeIncludingOnlyOriginalForTabular;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingOnlyOriginalForTabular));
 
         // Get the total size including both the original and archival tabular file sizes
         expectedSizeIncludingAllSizes = tabularArchivalSize + tabularOriginalSize + expectedSizeIncludingAllSizes;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
+
+        // Get the total size including both the original and archival tabular file sizes with search criteria
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, "text/plain", FileSearchCriteria.FileAccessStatus.Public.toString(), null, null, "test_", DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        // We exclude tabular sizes from the expected result since the search criteria filters by content type "text/plain" and search text "test_"
+        int expectedSizeIncludingAllSizesAndApplyingCriteria = testFileSize1 + testFileSize2;
+        getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
+                .body("data.storageSize", equalTo(expectedSizeIncludingAllSizesAndApplyingCriteria));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4421e9280b3..38cc44c8c0d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3450,10 +3450,26 @@ static Response deaccessionDataset(Integer datasetId, String version, String dea
                 .post("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
     }
 
-    static Response getDownloadSize(Integer datasetId, String version, String mode, String apiToken) {
-        return given()
+    static Response getDownloadSize(Integer datasetId, String version, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, String mode, String apiToken) {
+        RequestSpecification requestSpecification = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .queryParam("mode", mode)
+                .queryParam("mode", mode);
+        if (contentType != null) {
+            requestSpecification = requestSpecification.queryParam("contentType", contentType);
+        }
+        if (accessStatus != null) {
+            requestSpecification = requestSpecification.queryParam("accessStatus", accessStatus);
+        }
+        if (categoryName != null) {
+            requestSpecification = requestSpecification.queryParam("categoryName", categoryName);
+        }
+        if (tabularTagName != null) {
+            requestSpecification = requestSpecification.queryParam("tabularTagName", tabularTagName);
+        }
+        if (searchText != null) {
+            requestSpecification = requestSpecification.queryParam("searchText", searchText);
+        }
+        return requestSpecification
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/downloadsize");
     }
 }

From b6bcbf7cadcf8e7b2f05825836d155f6a589b710 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 16 Oct 2023 12:47:35 +0100
Subject: [PATCH 565/922] Added: getDownloadSize API endpoint deaccessioned
 dataset support

---
 .../harvard/iq/dataverse/api/Datasets.java    |  3 +-
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 41 +++++++++++++++----
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 26 ++++++++++--
 3 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 8605b4772f4..852dd18ee84 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2998,6 +2998,7 @@ public Response getDownloadSize(@Context ContainerRequestContext crc,
                                     @QueryParam("tabularTagName") String tabularTagName,
                                     @QueryParam("searchText") String searchText,
                                     @QueryParam("mode") String mode,
+                                    @QueryParam("includeDeaccessioned") boolean includeDeaccessioned,
                                     @Context UriInfo uriInfo,
                                     @Context HttpHeaders headers) {
 
@@ -3020,7 +3021,7 @@ public Response getDownloadSize(@Context ContainerRequestContext crc,
             } catch (IllegalArgumentException e) {
                 return error(Response.Status.BAD_REQUEST, "Invalid mode: " + mode);
             }
-            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers);
+            DatasetVersion datasetVersion = getDatasetVersionOrDie(req, version, findDatasetOrDie(dvIdtf), uriInfo, headers, includeDeaccessioned);
             long datasetStorageSize = datasetVersionFilesServiceBean.getFilesDownloadSize(datasetVersion, fileSearchCriteria, fileDownloadSizeMode);
             String message = MessageFormat.format(BundleUtil.getStringFromBundle("datasets.api.datasize.download"), datasetStorageSize);
             JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 66a67887405..e12de1e23cc 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3863,7 +3863,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         int expectedTextFilesStorageSize = testFileSize1 + testFileSize2;
 
         // Get the total size when there are no tabular files
-        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        Response getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedTextFilesStorageSize));
 
@@ -3878,7 +3878,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         Thread.sleep(2000);
 
         // Get the total size ignoring the original tabular file sizes
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Archival.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Archival.toString(), false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode());
 
         int actualSizeIgnoringOriginalTabularSizes = Integer.parseInt(getDownloadSizeResponse.getBody().jsonPath().getString("data.storageSize"));
@@ -3889,7 +3889,7 @@ public void getDownloadSize() throws IOException, InterruptedException {
         // Get the total size including only original sizes and ignoring archival sizes for tabular files
         int expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedTextFilesStorageSize;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingOnlyOriginalForTabular));
 
@@ -3897,13 +3897,13 @@ public void getDownloadSize() throws IOException, InterruptedException {
         int tabularArchivalSize = actualSizeIgnoringOriginalTabularSizes - expectedTextFilesStorageSize;
         int expectedSizeIncludingAllSizes = tabularArchivalSize + tabularOriginalSize + expectedTextFilesStorageSize;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
 
         // Get the total size sending invalid file download size mode
         String invalidMode = "invalidMode";
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, invalidMode, apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, invalidMode, false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode())
                 .body("message", equalTo("Invalid mode: " + invalidMode));
 
@@ -3917,22 +3917,47 @@ public void getDownloadSize() throws IOException, InterruptedException {
         // Get the total size including only original sizes and ignoring archival sizes for tabular files
         expectedSizeIncludingOnlyOriginalForTabular = tabularOriginalSize + expectedSizeIncludingOnlyOriginalForTabular;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.Original.toString(), false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingOnlyOriginalForTabular));
 
         // Get the total size including both the original and archival tabular file sizes
         expectedSizeIncludingAllSizes = tabularArchivalSize + tabularOriginalSize + expectedSizeIncludingAllSizes;
 
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), false, apiToken);
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
 
         // Get the total size including both the original and archival tabular file sizes with search criteria
-        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, "text/plain", FileSearchCriteria.FileAccessStatus.Public.toString(), null, null, "test_", DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), apiToken);
+        getDownloadSizeResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST, "text/plain", FileSearchCriteria.FileAccessStatus.Public.toString(), null, null, "test_", DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), false, apiToken);
         // We exclude tabular sizes from the expected result since the search criteria filters by content type "text/plain" and search text "test_"
         int expectedSizeIncludingAllSizesAndApplyingCriteria = testFileSize1 + testFileSize2;
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizesAndApplyingCriteria));
+
+        // Test Deaccessioned
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // includeDeaccessioned false
+        Response getVersionFileCountsResponseNoDeaccessioned = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST_PUBLISHED, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), false, apiToken);
+        getVersionFileCountsResponseNoDeaccessioned.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // includeDeaccessioned true
+        Response getVersionFileCountsResponseDeaccessioned = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST_PUBLISHED, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), true, apiToken);
+        getVersionFileCountsResponseDeaccessioned.then().assertThat().statusCode(OK.getStatusCode());
+
+        // Test that the dataset file counts for a deaccessioned dataset cannot be accessed by a guest
+        // By latest published version
+        Response getVersionFileCountsGuestUserResponse = UtilIT.getDownloadSize(datasetId, DS_VERSION_LATEST_PUBLISHED, null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), true, null);
+        getVersionFileCountsGuestUserResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+        // By specific version 1.0
+        getVersionFileCountsGuestUserResponse = UtilIT.getDownloadSize(datasetId, "1.0", null, null, null, null, null, DatasetVersionFilesServiceBean.FileDownloadSizeMode.All.toString(), true, null);
+        getVersionFileCountsGuestUserResponse.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 38cc44c8c0d..15350782fa1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3373,7 +3373,15 @@ static Response createFileEmbargo(Integer datasetId, Integer fileId, String date
                 .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo");
     }
 
-    static Response getVersionFileCounts(Integer datasetId, String version, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, boolean includeDeaccessioned, String apiToken) {
+    static Response getVersionFileCounts(Integer datasetId,
+                                         String version,
+                                         String contentType,
+                                         String accessStatus,
+                                         String categoryName,
+                                         String tabularTagName,
+                                         String searchText,
+                                         boolean includeDeaccessioned,
+                                         String apiToken) {
         RequestSpecification requestSpecification = given()
                 .queryParam("includeDeaccessioned", includeDeaccessioned);
         if (apiToken != null) {
@@ -3450,10 +3458,22 @@ static Response deaccessionDataset(Integer datasetId, String version, String dea
                 .post("/api/datasets/" + datasetId + "/versions/" + version + "/deaccession");
     }
 
-    static Response getDownloadSize(Integer datasetId, String version, String contentType, String accessStatus, String categoryName, String tabularTagName, String searchText, String mode, String apiToken) {
+    static Response getDownloadSize(Integer datasetId,
+                                    String version,
+                                    String contentType,
+                                    String accessStatus,
+                                    String categoryName,
+                                    String tabularTagName,
+                                    String searchText,
+                                    String mode,
+                                    boolean includeDeaccessioned,
+                                    String apiToken) {
         RequestSpecification requestSpecification = given()
-                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .queryParam("includeDeaccessioned", includeDeaccessioned)
                 .queryParam("mode", mode);
+        if (apiToken != null) {
+            requestSpecification.header(API_TOKEN_HTTP_HEADER, apiToken);
+        }
         if (contentType != null) {
             requestSpecification = requestSpecification.queryParam("contentType", contentType);
         }

From 794c5b64e6daa52e1d84fb2cb58468b5104d3161 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 16 Oct 2023 13:03:08 +0100
Subject: [PATCH 566/922] Added: extended docs for getDownloadSize API endpoint

---
 doc/sphinx-guides/source/api/native-api.rst | 61 ++++++++++++++++++++-
 1 file changed, 58 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 2e1a878dce8..98e2722ac5e 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -964,7 +964,7 @@ The fully expanded example above (without environment variables) looks like this
  
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files"
 
-This endpoint supports optional pagination, through the ``limit`` and ``offset`` query params:
+This endpoint supports optional pagination, through the ``limit`` and ``offset`` query parameters:
 
 .. code-block:: bash
 
@@ -1044,7 +1044,7 @@ Usage example:
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?includeDeaccessioned=true"
 
-.. note:: Keep in mind that you can combine all of the above query params depending on the results you are looking for.
+.. note:: Keep in mind that you can combine all of the above query parameters depending on the results you are looking for.
 
 Get File Counts in a Dataset
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1132,7 +1132,7 @@ Usage example:
 
 Please note that filtering values are case sensitive and must be correctly typed for the endpoint to recognize them.
 
-Keep in mind that you can combine all of the above query params depending on the results you are looking for.
+Keep in mind that you can combine all of the above query parameters depending on the results you are looking for.
 
 View Dataset Files and Folders as a Directory Index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1888,6 +1888,61 @@ Usage example:
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?mode=Archival"
 
+Category name filtering is also optionally supported. To return the size of all files available for download matching the requested category name.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?categoryName=Data"
+
+Tabular tag name filtering is also optionally supported. To return the size of all files available for download for which the requested tabular tag has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?tabularTagName=Survey"
+
+Content type filtering is also optionally supported. To return the size of all files available for download matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?contentType=image/png"
+
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the size of all files available for download that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?searchText=word"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Please note that filtering query parameters are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?includeDeaccessioned=true"
+
+.. note:: Keep in mind that you can combine all of the above query parameters depending on the results you are looking for.
+
 Submit a Dataset for Review
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From 28cd109303ec22cbb898f32f3141cc281f4c7c62 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 16 Oct 2023 13:10:38 +0100
Subject: [PATCH 567/922] Added: release notes for #9995

---
 ...adsize-with-criteria-and-deaccessioned-support.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md

diff --git a/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md b/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
new file mode 100644
index 00000000000..71c7aa3b516
--- /dev/null
+++ b/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
@@ -0,0 +1,12 @@
+Extended the getDownloadSize endpoint (/api/datasets/{id}/versions/{versionId}/files/downloadsize), including the following new features:
+
+- The endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned dataset versions when searching for versions to obtain the file total download size.
+
+
+- The endpoint now supports filtering by criteria. In particular, it accepts the following optional criteria query parameters:
+
+  - contentType
+  - accessStatus
+  - categoryName
+  - tabularTagName
+  - searchText

From ab237777309b90e299e584cff6995618bc378ebd Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 16 Oct 2023 13:22:30 +0100
Subject: [PATCH 568/922] Fixed: release notes

---
 ...-api-downloadsize-with-criteria-and-deaccessioned-support.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md b/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
index 71c7aa3b516..020224b2094 100644
--- a/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
+++ b/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
@@ -1,4 +1,4 @@
-Extended the getDownloadSize endpoint (/api/datasets/{id}/versions/{versionId}/files/downloadsize), including the following new features:
+Extended the getDownloadSize endpoint (/api/datasets/{id}/versions/{versionId}/downloadsize), including the following new features:
 
 - The endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned dataset versions when searching for versions to obtain the file total download size.
 

From e847ed04e87f16d5423bcfade38453fd1d959343 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Mon, 16 Oct 2023 16:53:58 +0100
Subject: [PATCH 569/922] Fixed: set label as second ordering column when
 ordering by content type

---
 .../harvard/iq/dataverse/DatasetVersionFilesServiceBean.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 2c14498caa9..701ff4474ea 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -293,7 +293,7 @@ private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> qu
                 query.orderBy(fileMetadata.dataFile.filesize.asc());
                 break;
             case Type:
-                query.orderBy(fileMetadata.dataFile.contentType.asc());
+                query.orderBy(fileMetadata.dataFile.contentType.asc(), fileMetadata.label.asc());
                 break;
             default:
                 query.orderBy(fileMetadata.label.asc());

From 4ad95697405512c16ec42b1d242ce620aec2436a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 16 Oct 2023 16:32:13 -0400
Subject: [PATCH 570/922] partial changes for permission mgmt, etc.

---
 .../harvard/iq/dataverse/api/Datasets.java    |  33 ++-
 .../dataverse/globus/GlobusServiceBean.java   | 218 +++++++++++-------
 .../iq/dataverse/settings/JvmSettings.java    |   1 +
 3 files changed, 155 insertions(+), 97 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 25839544ce9..d3ea1b80696 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3363,6 +3363,15 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
+        
+        JsonObject jsonObject = null;
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
+        } catch (Exception ex) {
+            logger.fine("Error parsing json: " + jsonData + " " + ex.getMessage());
+            return badRequest("Error parsing json body");
+
+        }
 
         //------------------------------------
         // (2b) Make sure dataset does not have package file
@@ -3396,7 +3405,7 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
         String requestUrl = SystemConfig.getDataverseSiteUrlStatic();
         
         // Async Call
-        globusService.globusUpload(jsonData, token, dataset, requestUrl, authUser);
+        globusService.globusUpload(jsonObject, token, dataset, requestUrl, authUser);
 
         return ok("Async call to Globus Upload started ");
 
@@ -3414,9 +3423,10 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
      */
     @POST
     @AuthRequired
-    @Path("{id}/allowGlobusUpload")
+    @Path("{id}/requestGlobusTransferPaths")
     @Consumes(MediaType.APPLICATION_JSON)
-    public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, String jsonBody
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, String jsonBody
     ) throws IOException, ExecutionException, InterruptedException {
 
 
@@ -3454,15 +3464,18 @@ public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathPar
             
         if (permissionSvc.requestOn(createDataverseRequest(authUser), dataset)
                 .canIssue(UpdateDatasetVersionCommand.class)) {
-
+            try {
             JsonObject params = JsonUtil.getJsonObject(jsonBody);
             String principal = params.getString("principal");
+            int numberOfPaths = params.getInt("numberOfFiles");
+            if(numberOfPaths <=0) {
+                return badRequest("numberOfFiles must be positive");
+            }
 
-            // Async Call
-            int status = globusService.givePermission("identity", principal, "rw", dataset);
-            switch (status) {
+            JsonObject response = globusService.requestAccessiblePaths(principal, dataset, numberOfPaths);
+            switch (response.getInt("status")) {
             case 201:
-                return ok("Permission Granted");
+                return ok(response.getJsonArray("paths"));
             case 400:
                 return badRequest("Unable to grant permission");
             case 409:
@@ -3470,6 +3483,10 @@ public Response allowGlobusUpload(@Context ContainerRequestContext crc, @PathPar
             default:
                 return error(null, "Unexpected error when granting permission");
             }
+            } catch (NullPointerException|ClassCastException e) {
+                return badRequest("Error retrieving principal and numberOfFiles from JSON request body");
+                
+            }
         } else {
             return forbidden("User doesn't have permission to upload to this dataset");
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index ad20b90971b..49572519696 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -1,7 +1,11 @@
 package edu.harvard.iq.dataverse.globus;
 
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.GsonBuilder;
+import com.nimbusds.oauth2.sdk.pkce.CodeVerifier;
+
 import edu.harvard.iq.dataverse.*;
 
 import jakarta.ejb.Asynchronous;
@@ -15,7 +19,9 @@
 import jakarta.json.JsonArray;
 import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonObject;
+import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonPatch;
+import jakarta.json.JsonValue;
 import jakarta.servlet.http.HttpServletRequest;
 
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
@@ -29,6 +35,8 @@
 import java.net.URLEncoder;
 import java.sql.Timestamp;
 import java.text.SimpleDateFormat;
+import java.time.Duration;
+import java.time.temporal.ChronoUnit;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
@@ -48,6 +56,7 @@
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -107,8 +116,10 @@ public void setUserTransferToken(String userTransferToken) {
         this.userTransferToken = userTransferToken;
     }
 
-    private ArrayList<String> checkPermissions(GlobusEndpoint endpoint, String principalType, String principal) throws MalformedURLException {
+    private String getRuleId(GlobusEndpoint endpoint, String principal, String permissions) throws MalformedURLException {
        
+        String principalType="identity";
+        
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access_list");
         MakeRequestResponse result = makeRequest(url, "Bearer",
                 endpoint.getClientToken(), "GET", null);
@@ -118,20 +129,22 @@ private ArrayList<String> checkPermissions(GlobusEndpoint endpoint, String princ
 
             for (int i = 0; i < al.getDATA().size(); i++) {
                 Permissions pr = al.getDATA().get(i);
+                
                 if ((pr.getPath().equals(endpoint.getBasePath() + "/") || pr.getPath().equals(endpoint.getBasePath()))
                         && pr.getPrincipalType().equals(principalType)
-                        && ((principal == null) || (principal != null && pr.getPrincipal().equals(principal)))) {
-                    ids.add(pr.getId());
+                        && ((principal == null) || (principal != null && pr.getPrincipal().equals(principal)))
+                        &&pr.getPermissions().equals(permissions)) {
+                    return pr.getId();
                 } else {
-                    logger.info(pr.getPath() + " === " + endpoint.getBasePath() + " == " + pr.getPrincipalType());
+                    logger.fine(pr.getPath() + " === " + endpoint.getBasePath() + " == " + pr.getPrincipalType());
                     continue;
                 }
             }
         }
-
-        return ids;
+        return null;
     }
-/*
+
+    /*
     public void updatePermision(AccessToken clientTokenUser, String directory, String principalType, String perm)
             throws MalformedURLException {
         if (directory != null && !directory.equals("")) {
@@ -165,47 +178,71 @@ public void updatePermision(AccessToken clientTokenUser, String directory, Strin
         }
     }
 */
-    public void deletePermission(String ruleId, Logger globusLogger) throws MalformedURLException {
-
-        if (ruleId.length() > 0) {
-            AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
-
-            globusLogger.info("Start deleting permissions.");
-            String globusEndpoint = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusEndpoint, "");
-
-            URL url = new URL(
-                    "https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/access/" + ruleId);
-            MakeRequestResponse result = makeRequest(url, "Bearer",
-                    clientTokenUser.getOtherTokens().get(0).getAccessToken(), "DELETE", null);
-            if (result.status != 200) {
-                globusLogger.warning("Cannot delete access rule " + ruleId);
-            } else {
-                globusLogger.info("Access rule " + ruleId + " was deleted successfully");
+    
+/** Call to delete a globus rule related to the specified dataset.
+ * 
+ * @param ruleId - Globus rule id - assumed to be associated with the dataset's file path (should not be called with a user specified rule id w/o further checking)
+ * @param datasetId - the id of the dataset associated with the rule
+ * @param globusLogger - a separate logger instance, may be null
+ */
+public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) {
+
+    if (ruleId.length() > 0) {
+        if (dataset != null) {
+            GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+            if (endpoint != null) {
+                String accessToken = endpoint.getClientToken();
+                if (globusLogger != null) {
+                    globusLogger.info("Start deleting permissions.");
+                }
+                try {
+                    URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId()
+                            + "/access/" + ruleId);
+                    MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "DELETE", null);
+                    if (result.status != 200) {
+                        if (globusLogger != null) {
+                            globusLogger.warning("Cannot delete access rule " + ruleId);
+                        } else {
+                            // When removed due to a cache ejection, we don't have a globusLogger
+                            logger.warning("Cannot delete access rule " + ruleId);
+                        }
+                    } else {
+                        if (globusLogger != null) {
+                            globusLogger.info("Access rule " + ruleId + " was deleted successfully");
+                        }
+                    }
+                } catch (MalformedURLException ex) {
+                    logger.log(Level.WARNING,
+                            "Failed to delete access rule " + ruleId + " on endpoint " + endpoint.getId(), ex);
+                }
             }
         }
-
     }
+}
 
-    public int givePermission(String principalType, String principal, String perm, Dataset dataset) throws MalformedURLException {
+    public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int numberOfPaths) {
 
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
-        ArrayList<?> rules = checkPermissions(endpoint, principalType, principal);
+        String principalType= "identity";
 
         Permissions permissions = new Permissions();
         permissions.setDATA_TYPE("access");
         permissions.setPrincipalType(principalType);
         permissions.setPrincipal(principal);
         permissions.setPath(endpoint.getBasePath() + "/");
-        permissions.setPermissions(perm);
+        permissions.setPermissions("rw");
 
         Gson gson = new GsonBuilder().create();
         MakeRequestResponse result = null;
-        if (rules.size() == 0) {
             logger.info("Start creating the rule");
+            JsonObjectBuilder response = Json.createObjectBuilder();
+
+            try {
             URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access");
             result = makeRequest(url, "Bearer", endpoint.getClientToken(), "POST",
                     gson.toJson(permissions));
 
+            response.add("status", result.status);
             switch (result.status) {
             case 400:
 
@@ -215,35 +252,50 @@ public int givePermission(String principalType, String principal, String perm, D
                 logger.warning("ACL already exists or Endpoint ACL already has the maximum number of access rules");
                 break;
             case 201:
-                JsonObject response = JsonUtil.getJsonObject(result.jsonResponse);
-                if (response != null && response.containsKey("access_id")) {
-                    permissions.setId(response.getString("access_id"));
-                    monitorTemporaryPermissions(permissions, endpoint);
+                JsonObject globusResponse = JsonUtil.getJsonObject(result.jsonResponse);
+                if (globusResponse != null && globusResponse.containsKey("access_id")) {
+                    permissions.setId(globusResponse.getString("access_id"));
+                    monitorTemporaryPermissions(permissions.getId(), dataset.getId());
                     logger.info("Access rule " + permissions.getId() + " was created successfully");
+                    JsonArrayBuilder pathArray = Json.createArrayBuilder();
+                    for(int i=0;i<numberOfPaths;i++) {
+                        pathArray.add(getUniqueFilePath(endpoint));
+                    
+                    }
+                    response.add("paths", pathArray.build());
+                    
+                } else {
+                    //Shouldn't happen!
+                    logger.warning("Access rule id not returned for dataset " + dataset.getId());
                 }
             }
-
-            return result.status;
-        } else {
-            logger.info("Start Updating the rule");
-            URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access/"
-                    + rules.get(0));
-            result = makeRequest(url, "Bearer", endpoint.getClientToken(), "PUT",
-                    gson.toJson(permissions));
-
-            if (result.status == 400) {
-                logger.severe("Path " + permissions.getPath() + " is not valid");
-            } else if (result.status == 409) {
-                logger.warning("ACL already exists or Endpoint ACL already has the maximum number of access rules");
+            } catch (MalformedURLException ex) {
+                //Misconfiguration
+                logger.warning("Failed to create access rule URL for " + endpoint.getId());
+                response.add("status", 500);
             }
-            logger.info("Result status " + result.status);
-            return result.status;
-        }
+            return response.build();
     }
 
-    private void monitorTemporaryPermissions(Permissions permissions, GlobusEndpoint endpoint) {
-        // TODO Auto-generated method stub
-        
+    private String getUniqueFilePath(GlobusEndpoint endpoint) {
+        // TODO See if generated identifier exists at globus endpoint
+        return endpoint.getBasePath() + "/" + FileUtil.generateStorageIdentifier();
+    }
+
+    //Single cache of open rules/permission requests
+    private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
+            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
+            .removalListener((ruleId, datasetId, cause) -> {
+                //Delete rules that expire
+                Dataset dataset = datasetSvc.find(datasetId);
+                deletePermission((String) ruleId, dataset, null);
+              })
+            
+            .build();
+    
+    
+    private void monitorTemporaryPermissions(String ruleId, long datasetId) {
+        rulesCache.put(ruleId, datasetId);
     }
 
     public boolean getSuccessfulTransfers(AccessToken clientTokenUser, String taskId) throws MalformedURLException {
@@ -468,6 +520,7 @@ private MakeRequestResponse findDirectory(String directory, String clientToken,
         return result;
     }
 
+    /*
     public boolean giveGlobusPublicPermissions(Dataset dataset)
             throws UnsupportedEncodingException, MalformedURLException {
 
@@ -478,20 +531,6 @@ public boolean giveGlobusPublicPermissions(Dataset dataset)
 
         if (status.status == 200) {
 
-            /*
-             * FilesList fl = parseJson(status.jsonResponse, FilesList.class, false);
-             * ArrayList<FileG> files = fl.getDATA(); if (files != null) { for (FileG file:
-             * files) { if (!file.getName().contains("cached") &&
-             * !file.getName().contains(".thumb")) { int perStatus =
-             * givePermission("all_authenticated_users", "", "r", clientTokenUser, directory
-             * + "/" + file.getName(), globusEndpoint); logger.info("givePermission status "
-             * + perStatus + " for " + file.getName()); if (perStatus == 409) {
-             * logger.info("Permissions already exist or limit was reached for " +
-             * file.getName()); } else if (perStatus == 400) {
-             * logger.info("No file in Globus " + file.getName()); } else if (perStatus !=
-             * 201) { logger.info("Cannot get permission for " + file.getName()); } } } }
-             */
-
             int perStatus = givePermission("all_authenticated_users", "", "r", dataset);
             logger.info("givePermission status " + perStatus);
             if (perStatus == 409) {
@@ -512,7 +551,8 @@ public boolean giveGlobusPublicPermissions(Dataset dataset)
 
         return true;
     }
-
+*/
+    
     // Generates the URL to launch the Globus app
     public String getGlobusAppUrlForDataset(Dataset d) {
         return getGlobusAppUrlForDataset(d, true, null);
@@ -572,7 +612,7 @@ public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken) {
     
     @Asynchronous
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
-    public void globusUpload(String jsonData, ApiToken token, Dataset dataset, String httpRequestUrl,
+    public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, String httpRequestUrl,
             AuthenticatedUser authUser) throws ExecutionException, InterruptedException, MalformedURLException {
 
         Integer countAll = 0;
@@ -606,33 +646,33 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
         // ToDo - use DataAccess methods?
         //String storageType = datasetIdentifier.substring(0, datasetIdentifier.indexOf("://") + 3);
         //datasetIdentifier = datasetIdentifier.substring(datasetIdentifier.indexOf("://") + 3);
+        
+        logger.fine("json: " + JsonUtil.prettyPrint(jsonData));
 
-        Thread.sleep(5000);
-
-        JsonObject jsonObject = null;
-        try (StringReader rdr = new StringReader(jsonData)) {
-            jsonObject = Json.createReader(rdr).readObject();
-        } catch (Exception jpe) {
-            jpe.printStackTrace();
-            logger.log(Level.SEVERE, "Error parsing dataset json. Json: {0}");
-        }
-        logger.info("json: " + JsonUtil.prettyPrint(jsonObject));
-
-        String taskIdentifier = jsonObject.getString("taskIdentifier");
+        String taskIdentifier = jsonData.getString("taskIdentifier");
 
-        String ruleId = "";
-        try {
-            ruleId = jsonObject.getString("ruleId");
-        } catch (NullPointerException npe) {
-            logger.warning("NPE for jsonData object");
-        }
+        String ruleId = null;
 
+        Thread.sleep(5000);
+        
         // globus task status check
         GlobusTask task = globusStatusCheck(taskIdentifier, globusLogger);
         String taskStatus = getTaskStatus(task);
 
-        if (ruleId.length() > 0) {
-            deletePermission(ruleId, globusLogger);
+        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+        
+        ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
+        if(ruleId!=null) {
+            Long datasetId = rulesCache.getIfPresent(ruleId);
+            if(datasetId!=null) {
+             
+            //Will delete rule
+            rulesCache.invalidate(ruleId);
+            } else {
+                //The cache already expired this rule, in which case it's delay not long enough, or we have some other problem
+                logger.warning("Rule " + ruleId + " not found in rulesCache");
+                deletePermission(ruleId, dataset, globusLogger);
+            }
         }
 
         // If success, switch to an EditInProgress lock - do this before removing the
@@ -674,7 +714,7 @@ public void globusUpload(String jsonData, ApiToken token, Dataset dataset, Strin
                 //
 
                 List<String> inputList = new ArrayList<String>();
-                JsonArray filesJsonArray = jsonObject.getJsonArray("files");
+                JsonArray filesJsonArray = jsonData.getJsonArray("files");
 
                 if (filesJsonArray != null) {
                     String datasetIdentifier = dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
@@ -905,7 +945,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         String taskStatus = getTaskStatus(task);
 
         if (ruleId.length() > 0) {
-            deletePermission(ruleId, globusLogger);
+            deletePermission(ruleId, dataset, globusLogger);
         }
 
         if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index b4807372b69..f8abe505dca 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -48,6 +48,7 @@ public enum JvmSettings {
     SCOPE_FILES(PREFIX, "files"),
     FILES_DIRECTORY(SCOPE_FILES, "directory"),
     GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"),
+    GLOBUS_RULES_CACHE_MAXAGE(SCOPE_FILES, "globus-rules-cache-maxage"),
     FILES(SCOPE_FILES),
     BASE_URL(FILES, "base-url"),
     GLOBUS_TOKEN(FILES, "globus-token"),

From 30395309689949a3fc633e3be5fa4c30cc1f27cd Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 16 Oct 2023 16:33:02 -0400
Subject: [PATCH 571/922] check driver type not id

---
 .../java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index f4cc7d40120..3bc83538679 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -606,7 +606,7 @@ public static String getDriverPrefix(String driverId) {
     }
     
     public static boolean isDirectUploadEnabled(String driverId) {
-        return (DataAccess.S3.equals(driverId) && Boolean.parseBoolean(System.getProperty("dataverse.files." + DataAccess.S3 + ".upload-redirect"))) ||
+        return (System.getProperty("dataverse.files." + driverId + ".type").equals(DataAccess.S3) && Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect"))) ||
             Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
     }
     

From 635d345df3b71484b827668946b48b017420eedd Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 16 Oct 2023 16:33:02 -0400
Subject: [PATCH 572/922] check driver type not id

---
 .../java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index 00db98e894e..d33f8f5e5bd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -606,7 +606,7 @@ public static String getDriverPrefix(String driverId) {
     }
     
     public static boolean isDirectUploadEnabled(String driverId) {
-        return (DataAccess.S3.equals(driverId) && Boolean.parseBoolean(System.getProperty("dataverse.files." + DataAccess.S3 + ".upload-redirect"))) ||
+        return (System.getProperty("dataverse.files." + driverId + ".type").equals(DataAccess.S3) && Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect"))) ||
             Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
     }
     

From 48144a24cb200e285b5419ab29865293eac17e54 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 16 Oct 2023 17:00:51 -0400
Subject: [PATCH 573/922] adding extra logic to skip things like facets and
 highlights in searches, unless specifically requested. (#9635)

---
 .../search/SearchIncludeFragment.java         |  91 ++++--
 .../dataverse/search/SearchServiceBean.java   | 308 +++++++++++-------
 2 files changed, 249 insertions(+), 150 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 2ce06541afa..1e42958fe4e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -120,7 +120,6 @@ public class SearchIncludeFragment implements java.io.Serializable {
     private Long facetCountDatasets = 0L;
     private Long facetCountFiles = 0L;
     Map<String, Long> previewCountbyType = new HashMap<>();
-    private SolrQueryResponse solrQueryResponseAllTypes;
     private String sortField;
     private SortOrder sortOrder;
     private String currentSort;
@@ -132,6 +131,7 @@ public class SearchIncludeFragment implements java.io.Serializable {
     Map<String, String> datasetfieldFriendlyNamesBySolrField = new HashMap<>();
     Map<String, String> staticSolrFieldFriendlyNamesBySolrField = new HashMap<>();
     private boolean solrIsDown = false;
+    private boolean solrIsOverloaded = false; 
     private Map<String, Integer> numberOfFacets = new HashMap<>();
 //    private boolean showUnpublished;
     List<String> filterQueriesDebug = new ArrayList<>();
@@ -279,6 +279,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
 
 
         SolrQueryResponse solrQueryResponse = null;
+        SolrQueryResponse solrQueryResponseSecondPass = null;
 
         List<String> filterQueriesFinal = new ArrayList<>();
         
@@ -311,18 +312,11 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         String[] parts = selectedTypesString.split(":");
         selectedTypesList.addAll(Arrays.asList(parts));
 
-        List<String> filterQueriesFinalAllTypes = new ArrayList<>();
-        String[] arr = selectedTypesList.toArray(new String[selectedTypesList.size()]);
-        selectedTypesHumanReadable = combine(arr, " OR ");
-        if (!selectedTypesHumanReadable.isEmpty()) {
-            typeFilterQuery = SearchFields.TYPE + ":(" + selectedTypesHumanReadable + ")";
-        }
+        
         
         filterQueriesFinal.addAll(filterQueries);
-        filterQueriesFinalAllTypes.addAll(filterQueriesFinal); 
 
-        String allTypesFilterQuery = SearchFields.TYPE + ":(dataverses OR datasets OR files)";
-        filterQueriesFinalAllTypes.add(allTypesFilterQuery);
+        
         filterQueriesFinal.add(typeFilterQuery);
 
         if (page <= 1) {
@@ -363,10 +357,60 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             // This 2nd search() is for populating the "type" ("dataverse", "dataset", "file") facets: -- L.A. 
             // (why exactly do we need it, again?)
             // To get the counts we display in the types facets particulary for unselected types - SEK 08/25/2021
-            solrQueryResponseAllTypes = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalAllTypes, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null);
-            if (solrQueryResponse.hasError()){
-                logger.info(solrQueryResponse.getError());
-                setSolrErrorEncountered(true);
+            // Sure, but we should not waste resources here. We will try to save 
+            // solr some extra work and a) only run this second query IF there is 
+            // one or more unselected type facets; and b) drop all the extra 
+            // parameters from this second query - such as facets and highlights -
+            // that we do not actually need for the purposes of finding these 
+            // extra numbers. -- L.A. 10/16/2023
+            
+            // populate preview counts: https://redmine.hmdc.harvard.edu/issues/3560
+            previewCountbyType.put(BundleUtil.getStringFromBundle("dataverses"), -1L);
+            previewCountbyType.put(BundleUtil.getStringFromBundle("datasets"), -1L);
+            previewCountbyType.put(BundleUtil.getStringFromBundle("files"), -1L);
+            
+            
+            // This will populate the type facet counts for the types that are 
+            // currently selected on the collection page:
+            for (FacetCategory facetCategory : solrQueryResponse.getTypeFacetCategories()) {
+                for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                    previewCountbyType.put(facetLabel.getName(), facetLabel.getCount());
+                }
+            }
+            
+            if (selectedTypesList.size() < 3) {
+                // If some types are NOT currently selected, we will need to 
+                // run another query to obtain the numbers of the unselected types:
+                
+                List<String> filterQueriesFinalSecondPass = new ArrayList<>();
+                filterQueriesFinalSecondPass.addAll(filterQueriesFinal);
+                
+                List<String> selectedTypesListSecondPass = new ArrayList<>();
+                
+                for (String dvObjectType : previewCountbyType.keySet()) {
+                    if (previewCountbyType.get(dvObjectType) == -1) {
+                        selectedTypesListSecondPass.add(dvObjectType);
+                    }
+                }
+                
+                String[] arr = selectedTypesListSecondPass.toArray(new String[selectedTypesListSecondPass.size()]);
+                filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
+                
+                if (solrQueryResponseSecondPass != null) {
+
+                    solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null);
+                    if (solrQueryResponseSecondPass.hasError()) {
+                        logger.info(solrQueryResponse.getError());
+                        setSolrErrorEncountered(true);
+                    }
+
+                    // And now populate the remaining type facets:
+                    for (FacetCategory facetCategory : solrQueryResponseSecondPass.getTypeFacetCategories()) {
+                        for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                            previewCountbyType.put(facetLabel.getName(), facetLabel.getCount());
+                        }
+                    }
+                }
             }
             
         } catch (SearchException ex) {
@@ -446,17 +490,6 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 }
             }
 
-            // populate preview counts: https://redmine.hmdc.harvard.edu/issues/3560
-            previewCountbyType.put(BundleUtil.getStringFromBundle("dataverses"), 0L);
-            previewCountbyType.put(BundleUtil.getStringFromBundle("datasets"), 0L);
-            previewCountbyType.put(BundleUtil.getStringFromBundle("files"), 0L);
-            if (solrQueryResponseAllTypes != null) {
-                for (FacetCategory facetCategory : solrQueryResponseAllTypes.getTypeFacetCategories()) {
-                    for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
-                        previewCountbyType.put(facetLabel.getName(), facetLabel.getCount());
-                    }
-                }
-            }
             
             setDisplayCardValues();
             
@@ -1020,6 +1053,14 @@ public boolean isSolrIsDown() {
     public void setSolrIsDown(boolean solrIsDown) {
         this.solrIsDown = solrIsDown;
     }
+    
+    public boolean isSolrOverloaded() {
+        return solrIsOverloaded;
+    }
+    
+    public void setSolrIsOverloaded(boolean solrIsOverloaded) {
+        this.solrIsOverloaded = solrIsOverloaded; 
+    }
 
     public boolean isRootDv() {
         return rootDv;
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 44976d232c2..aa2948eb8cb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -101,7 +101,7 @@ public class SearchServiceBean {
     public SolrQueryResponse search(DataverseRequest dataverseRequest, List<Dataverse> dataverses, String query, List<String> filterQueries, String sortField, String sortOrder, int paginationStart, boolean onlyDatatRelatedToMe, int numResultsPerPage) throws SearchException {
         return search(dataverseRequest, dataverses, query, filterQueries, sortField, sortOrder, paginationStart, onlyDatatRelatedToMe, numResultsPerPage, true, null, null);
     }
-
+    
     /**
      * Import note: "onlyDatatRelatedToMe" relies on filterQueries for providing
      * access to Private Data for the correct user
@@ -122,6 +122,41 @@ public SolrQueryResponse search(DataverseRequest dataverseRequest, List<Datavers
      * @param retrieveEntities - look up dvobject entities with .find() (potentially expensive!)
      * @param geoPoint e.g. "35,15"
      * @param geoRadius e.g. "5"
+
+     * @return
+     * @throws SearchException
+     */
+    public SolrQueryResponse search(
+            DataverseRequest dataverseRequest, 
+            List<Dataverse> dataverses, 
+            String query, 
+            List<String> filterQueries, 
+            String sortField, 
+            String sortOrder, 
+            int paginationStart, 
+            boolean onlyDatatRelatedToMe, 
+            int numResultsPerPage,
+            boolean retrieveEntities,
+            String geoPoint,
+            String geoRadius) throws SearchException {
+        return search(dataverseRequest, dataverses, query, filterQueries, sortField, sortOrder, paginationStart, onlyDatatRelatedToMe, numResultsPerPage, true, null, null, true, true);
+    }
+
+    /**
+     * @param dataverseRequest
+     * @param dataverses
+     * @param query
+     * @param filterQueries
+     * @param sortField
+     * @param sortOrder
+     * @param paginationStart
+     * @param onlyDatatRelatedToMe
+     * @param numResultsPerPage
+     * @param retrieveEntities - look up dvobject entities with .find() (potentially expensive!)
+     * @param geoPoint e.g. "35,15"
+     * @param geoRadius e.g. "5"
+     * @param addFacets boolean
+     * @param addHighlights boolean
      * @return
      * @throws SearchException
      */
@@ -136,7 +171,9 @@ public SolrQueryResponse search(
             int numResultsPerPage,
             boolean retrieveEntities,
             String geoPoint,
-            String geoRadius
+            String geoRadius,
+            boolean addFacets,
+            boolean addHighlights
     ) throws SearchException {
 
         if (paginationStart < 0) {
@@ -157,56 +194,62 @@ public SolrQueryResponse search(
 //            solrQuery.setSort(sortClause);
 //        }
 //        solrQuery.setSort(sortClause);
-        solrQuery.setHighlight(true).setHighlightSnippets(1);
-        Integer fragSize = systemConfig.getSearchHighlightFragmentSize();
-        if (fragSize != null) {
-            solrQuery.setHighlightFragsize(fragSize);
-        }
-        solrQuery.setHighlightSimplePre("<span class=\"search-term-match\">");
-        solrQuery.setHighlightSimplePost("</span>");
+
+        List<DatasetFieldType> datasetFields = datasetFieldService.findAllOrderedById();
         Map<String, String> solrFieldsToHightlightOnMap = new HashMap<>();
-        // TODO: Do not hard code "Name" etc as English here.
-        solrFieldsToHightlightOnMap.put(SearchFields.NAME, "Name");
-        solrFieldsToHightlightOnMap.put(SearchFields.AFFILIATION, "Affiliation");
-        solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_FRIENDLY, "File Type");
-        solrFieldsToHightlightOnMap.put(SearchFields.DESCRIPTION, "Description");
-        solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NAME, "Variable Name");
-        solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_LABEL, "Variable Label");
-        solrFieldsToHightlightOnMap.put(SearchFields.LITERAL_QUESTION, BundleUtil.getStringFromBundle("search.datasets.literalquestion"));
-        solrFieldsToHightlightOnMap.put(SearchFields.INTERVIEW_INSTRUCTIONS, BundleUtil.getStringFromBundle("search.datasets.interviewinstructions"));
-        solrFieldsToHightlightOnMap.put(SearchFields.POST_QUESTION, BundleUtil.getStringFromBundle("search.datasets.postquestion"));
-        solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_UNIVERSE, BundleUtil.getStringFromBundle("search.datasets.variableuniverse"));
-        solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NOTES, BundleUtil.getStringFromBundle("search.datasets.variableNotes"));
-
-        solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_SEARCHABLE, "File Type");
-        solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PUBLICATION_DATE, "Publication Year");
-        solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.datasets.persistentId"));
-        solrFieldsToHightlightOnMap.put(SearchFields.FILE_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.files.persistentId"));
-        /**
-         * @todo Dataverse subject and affiliation should be highlighted but
-         * this is commented out right now because the "friendly" names are not
-         * being shown on the dataverse cards. See also
-         * https://github.com/IQSS/dataverse/issues/1431
-         */
+        if (addHighlights) {
+            solrQuery.setHighlight(true).setHighlightSnippets(1);
+            Integer fragSize = systemConfig.getSearchHighlightFragmentSize();
+            if (fragSize != null) {
+                solrQuery.setHighlightFragsize(fragSize);
+            }
+            solrQuery.setHighlightSimplePre("<span class=\"search-term-match\">");
+            solrQuery.setHighlightSimplePost("</span>");
+
+            // TODO: Do not hard code "Name" etc as English here.
+            solrFieldsToHightlightOnMap.put(SearchFields.NAME, "Name");
+            solrFieldsToHightlightOnMap.put(SearchFields.AFFILIATION, "Affiliation");
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_FRIENDLY, "File Type");
+            solrFieldsToHightlightOnMap.put(SearchFields.DESCRIPTION, "Description");
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NAME, "Variable Name");
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_LABEL, "Variable Label");
+            solrFieldsToHightlightOnMap.put(SearchFields.LITERAL_QUESTION, BundleUtil.getStringFromBundle("search.datasets.literalquestion"));
+            solrFieldsToHightlightOnMap.put(SearchFields.INTERVIEW_INSTRUCTIONS, BundleUtil.getStringFromBundle("search.datasets.interviewinstructions"));
+            solrFieldsToHightlightOnMap.put(SearchFields.POST_QUESTION, BundleUtil.getStringFromBundle("search.datasets.postquestion"));
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_UNIVERSE, BundleUtil.getStringFromBundle("search.datasets.variableuniverse"));
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NOTES, BundleUtil.getStringFromBundle("search.datasets.variableNotes"));
+
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_SEARCHABLE, "File Type");
+            solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PUBLICATION_DATE, "Publication Year");
+            solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.datasets.persistentId"));
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.files.persistentId"));
+            /**
+             * @todo Dataverse subject and affiliation should be highlighted but
+             * this is commented out right now because the "friendly" names are
+             * not being shown on the dataverse cards. See also
+             * https://github.com/IQSS/dataverse/issues/1431
+             */
 //        solrFieldsToHightlightOnMap.put(SearchFields.DATAVERSE_SUBJECT, "Subject");
 //        solrFieldsToHightlightOnMap.put(SearchFields.DATAVERSE_AFFILIATION, "Affiliation");
-        /**
-         * @todo: show highlight on file card?
-         * https://redmine.hmdc.harvard.edu/issues/3848
-         */
-        solrFieldsToHightlightOnMap.put(SearchFields.FILENAME_WITHOUT_EXTENSION, "Filename Without Extension");
-        solrFieldsToHightlightOnMap.put(SearchFields.FILE_TAG_SEARCHABLE, "File Tag");
-        List<DatasetFieldType> datasetFields = datasetFieldService.findAllOrderedById();
-        for (DatasetFieldType datasetFieldType : datasetFields) {
-            String solrField = datasetFieldType.getSolrField().getNameSearchable();
-            String displayName = datasetFieldType.getDisplayName();
-            solrFieldsToHightlightOnMap.put(solrField, displayName);
-        }
-        for (Map.Entry<String, String> entry : solrFieldsToHightlightOnMap.entrySet()) {
-            String solrField = entry.getKey();
-            // String displayName = entry.getValue();
-            solrQuery.addHighlightField(solrField);
+            /**
+             * @todo: show highlight on file card?
+             * https://redmine.hmdc.harvard.edu/issues/3848
+             */
+            solrFieldsToHightlightOnMap.put(SearchFields.FILENAME_WITHOUT_EXTENSION, "Filename Without Extension");
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TAG_SEARCHABLE, "File Tag");
+
+            for (DatasetFieldType datasetFieldType : datasetFields) {
+                String solrField = datasetFieldType.getSolrField().getNameSearchable();
+                String displayName = datasetFieldType.getDisplayName();
+                solrFieldsToHightlightOnMap.put(solrField, displayName);
+            }
+            for (Map.Entry<String, String> entry : solrFieldsToHightlightOnMap.entrySet()) {
+                String solrField = entry.getKey();
+                // String displayName = entry.getValue();
+                solrQuery.addHighlightField(solrField);
+            }
         }
+        
         solrQuery.setParam("fl", "*,score");
         solrQuery.setParam("qt", "/select");
         solrQuery.setParam("facet", "true");
@@ -214,6 +257,8 @@ public SolrQueryResponse search(
          * @todo: do we need facet.query?
          */
         solrQuery.setParam("facet.query", "*");
+        solrQuery.addFacetField(SearchFields.TYPE); // this one is always performed
+
         for (String filterQuery : filterQueries) {
             solrQuery.addFilterQuery(filterQuery);
         }
@@ -223,70 +268,73 @@ public SolrQueryResponse search(
             // See https://solr.apache.org/guide/8_11/spatial-search.html#bbox
             solrQuery.addFilterQuery("{!bbox sfield=" + SearchFields.GEOLOCATION + "}");
         }
+        
+        List<DataverseMetadataBlockFacet> metadataBlockFacets = new LinkedList<>();
 
-        // -----------------------------------
-        // Facets to Retrieve
-        // -----------------------------------
-        solrQuery.addFacetField(SearchFields.METADATA_TYPES);
-//        solrQuery.addFacetField(SearchFields.HOST_DATAVERSE);
-//        solrQuery.addFacetField(SearchFields.AUTHOR_STRING);
-        solrQuery.addFacetField(SearchFields.DATAVERSE_CATEGORY);
-        solrQuery.addFacetField(SearchFields.METADATA_SOURCE);
-//        solrQuery.addFacetField(SearchFields.AFFILIATION);
-        solrQuery.addFacetField(SearchFields.PUBLICATION_YEAR);
-//        solrQuery.addFacetField(SearchFields.CATEGORY);
-//        solrQuery.addFacetField(SearchFields.FILE_TYPE_MIME);
-//        solrQuery.addFacetField(SearchFields.DISTRIBUTOR);
-//        solrQuery.addFacetField(SearchFields.KEYWORD);
-        /**
-         * @todo when a new method on datasetFieldService is available
-         * (retrieveFacetsByDataverse?) only show the facets that the dataverse
-         * in question wants to show (and in the right order):
-         * https://redmine.hmdc.harvard.edu/issues/3490
-         *
-         * also, findAll only returns advancedSearchField = true... we should
-         * probably introduce the "isFacetable" boolean rather than caring about
-         * if advancedSearchField is true or false
-         *
-         */
+        if (addFacets) {
+            // -----------------------------------
+            // Facets to Retrieve
+            // -----------------------------------
+            solrQuery.addFacetField(SearchFields.METADATA_TYPES);
+            solrQuery.addFacetField(SearchFields.DATAVERSE_CATEGORY);
+            solrQuery.addFacetField(SearchFields.METADATA_SOURCE);
+            solrQuery.addFacetField(SearchFields.PUBLICATION_YEAR);
+            /**
+             * @todo when a new method on datasetFieldService is available
+             * (retrieveFacetsByDataverse?) only show the facets that the
+             * dataverse in question wants to show (and in the right order):
+             * https://redmine.hmdc.harvard.edu/issues/3490
+             *
+             * also, findAll only returns advancedSearchField = true... we
+             * should probably introduce the "isFacetable" boolean rather than
+             * caring about if advancedSearchField is true or false
+             *
+             */
 
-        List<DataverseMetadataBlockFacet> metadataBlockFacets = new LinkedList<>();
+            if (dataverses != null) {
+                for (Dataverse dataverse : dataverses) {
+                    if (dataverse != null) {
+                        for (DataverseFacet dataverseFacet : dataverse.getDataverseFacets()) {
+                            DatasetFieldType datasetField = dataverseFacet.getDatasetFieldType();
+                            solrQuery.addFacetField(datasetField.getSolrField().getNameFacetable());
+                        }
+                        // Get all metadata block facets configured to be displayed
+                        metadataBlockFacets.addAll(dataverse.getMetadataBlockFacets());
+                    }
+                }
+            }
+            
+            solrQuery.addFacetField(SearchFields.FILE_TYPE);
+            /**
+            * @todo: hide the extra line this shows in the GUI... at least it's
+            * last...
+            */
+            solrQuery.addFacetField(SearchFields.FILE_TAG);
+            if (!systemConfig.isPublicInstall()) {
+                solrQuery.addFacetField(SearchFields.ACCESS);
+            }
+        }
+
+        
         //I'm not sure if just adding null here is good for hte permissions system... i think it needs something
         if(dataverses != null) {
             for(Dataverse dataverse : dataverses) {
                 // -----------------------------------
                 // PERMISSION FILTER QUERY
                 // -----------------------------------
-                String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, dataverse, onlyDatatRelatedToMe);
+                String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, dataverse, onlyDatatRelatedToMe, addFacets);
                 if (permissionFilterQuery != null) {
                     solrQuery.addFilterQuery(permissionFilterQuery);
                 }
-                if (dataverse != null) {
-                    for (DataverseFacet dataverseFacet : dataverse.getDataverseFacets()) {
-                        DatasetFieldType datasetField = dataverseFacet.getDatasetFieldType();
-                        solrQuery.addFacetField(datasetField.getSolrField().getNameFacetable());
-                    }
-                    // Get all metadata block facets configured to be displayed
-                    metadataBlockFacets.addAll(dataverse.getMetadataBlockFacets());
-                }
             }
         } else {
-            String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, null, onlyDatatRelatedToMe);
+            String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, null, onlyDatatRelatedToMe, addFacets);
             if (permissionFilterQuery != null) {
                 solrQuery.addFilterQuery(permissionFilterQuery);
             }
         }
 
-        solrQuery.addFacetField(SearchFields.FILE_TYPE);
-        /**
-         * @todo: hide the extra line this shows in the GUI... at least it's
-         * last...
-         */
-        solrQuery.addFacetField(SearchFields.TYPE);
-        solrQuery.addFacetField(SearchFields.FILE_TAG);
-        if (!systemConfig.isPublicInstall()) {
-            solrQuery.addFacetField(SearchFields.ACCESS);
-        }
+        
         /**
          * @todo: do sanity checking... throw error if negative
          */
@@ -416,34 +464,44 @@ public SolrQueryResponse search(
             Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID);
             
             List<String> matchedFields = new ArrayList<>();
-            List<Highlight> highlights = new ArrayList<>();
-            Map<SolrField, Highlight> highlightsMap = new HashMap<>();
-            Map<SolrField, List<String>> highlightsMap2 = new HashMap<>();
-            Map<String, Highlight> highlightsMap3 = new HashMap<>();
-            if (queryResponse.getHighlighting().get(id) != null) {
-                for (Map.Entry<String, String> entry : solrFieldsToHightlightOnMap.entrySet()) {
-                    String field = entry.getKey();
-                    String displayName = entry.getValue();
-
-                    List<String> highlightSnippets = queryResponse.getHighlighting().get(id).get(field);
-                    if (highlightSnippets != null) {
-                        matchedFields.add(field);
-                        /**
-                         * @todo only SolrField.SolrType.STRING? that's not
-                         * right... knit the SolrField object more into the
-                         * highlighting stuff
-                         */
-                        SolrField solrField = new SolrField(field, SolrField.SolrType.STRING, true, true);
-                        Highlight highlight = new Highlight(solrField, highlightSnippets, displayName);
-                        highlights.add(highlight);
-                        highlightsMap.put(solrField, highlight);
-                        highlightsMap2.put(solrField, highlightSnippets);
-                        highlightsMap3.put(field, highlight);
+            
+            SolrSearchResult solrSearchResult = new SolrSearchResult(query, name);
+            
+            if (addHighlights) {
+                List<Highlight> highlights = new ArrayList<>();
+                Map<SolrField, Highlight> highlightsMap = new HashMap<>();
+                Map<SolrField, List<String>> highlightsMap2 = new HashMap<>();
+                Map<String, Highlight> highlightsMap3 = new HashMap<>();
+                if (queryResponse.getHighlighting().get(id) != null) {
+                    for (Map.Entry<String, String> entry : solrFieldsToHightlightOnMap.entrySet()) {
+                        String field = entry.getKey();
+                        String displayName = entry.getValue();
+
+                        List<String> highlightSnippets = queryResponse.getHighlighting().get(id).get(field);
+                        if (highlightSnippets != null) {
+                            matchedFields.add(field);
+                            /**
+                             * @todo only SolrField.SolrType.STRING? that's not
+                             * right... knit the SolrField object more into the
+                             * highlighting stuff
+                             */
+                            SolrField solrField = new SolrField(field, SolrField.SolrType.STRING, true, true);
+                            Highlight highlight = new Highlight(solrField, highlightSnippets, displayName);
+                            highlights.add(highlight);
+                            highlightsMap.put(solrField, highlight);
+                            highlightsMap2.put(solrField, highlightSnippets);
+                            highlightsMap3.put(field, highlight);
+                        }
                     }
+
                 }
 
+                solrSearchResult.setHighlightsAsList(highlights);
+                solrSearchResult.setHighlightsMap(highlightsMap);
+                solrSearchResult.setHighlightsAsMap(highlightsMap3);
             }
-            SolrSearchResult solrSearchResult = new SolrSearchResult(query, name);
+            
+            
             /**
              * @todo put all this in the constructor?
              */
@@ -470,9 +528,7 @@ public SolrQueryResponse search(
             solrSearchResult.setNameSort(nameSort);
             solrSearchResult.setReleaseOrCreateDate(release_or_create_date);
             solrSearchResult.setMatchedFields(matchedFields);
-            solrSearchResult.setHighlightsAsList(highlights);
-            solrSearchResult.setHighlightsMap(highlightsMap);
-            solrSearchResult.setHighlightsAsMap(highlightsMap3);
+            
             Map<String, String> parent = new HashMap<>();
             String description = (String) solrDocument.getFieldValue(SearchFields.DESCRIPTION);
             solrSearchResult.setDescriptionNoSnippet(description);
@@ -863,7 +919,7 @@ public String getCapitalizedName(String name) {
      *
      * @return
      */
-    private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQuery solrQuery, Dataverse dataverse, boolean onlyDatatRelatedToMe) {
+    private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQuery solrQuery, Dataverse dataverse, boolean onlyDatatRelatedToMe, boolean addFacets) {
 
         User user = dataverseRequest.getUser();
         if (user == null) {
@@ -922,9 +978,11 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
 
         AuthenticatedUser au = (AuthenticatedUser) user;
 
-        // Logged in user, has publication status facet
-        //
-        solrQuery.addFacetField(SearchFields.PUBLICATION_STATUS);
+        if (addFacets) {
+            // Logged in user, has publication status facet
+            //
+            solrQuery.addFacetField(SearchFields.PUBLICATION_STATUS);
+        }
 
         // ----------------------------------------------------
         // (3) Is this a Super User?

From 6cda2fcec8fd2ebfef8b9a11a907628ec7d4b1b2 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 17 Oct 2023 10:00:06 +0200
Subject: [PATCH 574/922] Remove StringReader in deaccession endpoint

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 7d50d5e0e70..f292f63c1ff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3974,8 +3974,8 @@ public Response deaccessionDataset(@Context ContainerRequestContext crc, @PathPa
         }
         return response(req -> {
             DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, false);
-            try (StringReader stringReader = new StringReader(jsonBody)) {
-                JsonObject jsonObject = Json.createReader(stringReader).readObject();
+            try {
+                JsonObject jsonObject = JsonUtil.getJsonObject(jsonBody);
                 datasetVersion.setVersionNote(jsonObject.getString("deaccessionReason"));
                 String deaccessionForwardURL = jsonObject.getString("deaccessionForwardURL", null);
                 if (deaccessionForwardURL != null) {

From 5802bf61fe92aae50d8764ee8df4dd8011292f22 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 17 Oct 2023 11:02:12 +0200
Subject: [PATCH 575/922] Lookup dataverse by alias or ID

---
 .../java/edu/harvard/iq/dataverse/api/Dataverses.java    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 30c14535251..47f6468bfac 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -1194,10 +1194,15 @@ public Response getGuestbookResponsesByDataverse(@Context ContainerRequestContex
             public void write(OutputStream os) throws IOException,
                     WebApplicationException {
 
-                Dataverse dv = dataverseService.findByAlias(dvIdtf);
+                Dataverse dv;
+                try {
+                    dv = findDataverseOrDie(dvIdtf);
+                } catch (WrappedResponse wr) {
+                    throw new WebApplicationException(wr.getResponse());
+                }
                 Map<Integer, Object> customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId);
                 Map<Integer, String> datasetTitles = guestbookResponseService.mapDatasetTitles(dv.getId());
-                
+
                 List<Object[]> guestbookResults = guestbookResponseService.getGuestbookResults(dv.getId(), gbId);
                 os.write("Guestbook, Dataset, Dataset PID, Date, Type, File Name, File Id, File PID, User Name, Email, Institution, Position, Custom Questions\n".getBytes());
                 for (Object[] result : guestbookResults) {

From f2770fb2f107b4e75701d2baf5f4f52b51abf709 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 17 Oct 2023 16:10:02 +0200
Subject: [PATCH 576/922] Do not look up the Dataverse twice

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 .../java/edu/harvard/iq/dataverse/api/Dataverses.java  | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 47f6468bfac..76cfa8ef764 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -1173,8 +1173,9 @@ public Response getGroupByOwnerAndAliasInOwner(@Context ContainerRequestContext
     public Response getGuestbookResponsesByDataverse(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf,
             @QueryParam("guestbookId") Long gbId, @Context HttpServletResponse response) {
 
+        Dataverse dv;
         try {
-            Dataverse dv = findDataverseOrDie(dvIdtf);
+            dv = findDataverseOrDie(dvIdtf);
             User u = getRequestUser(crc);
             DataverseRequest req = createDataverseRequest(u);
             if (permissionSvc.request(req)
@@ -1194,12 +1195,6 @@ public Response getGuestbookResponsesByDataverse(@Context ContainerRequestContex
             public void write(OutputStream os) throws IOException,
                     WebApplicationException {
 
-                Dataverse dv;
-                try {
-                    dv = findDataverseOrDie(dvIdtf);
-                } catch (WrappedResponse wr) {
-                    throw new WebApplicationException(wr.getResponse());
-                }
                 Map<Integer, Object> customQandAs = guestbookResponseService.mapCustomQuestionAnswersAsStrings(dv.getId(), gbId);
                 Map<Integer, String> datasetTitles = guestbookResponseService.mapDatasetTitles(dv.getId());
 
@@ -1208,7 +1203,6 @@ public void write(OutputStream os) throws IOException,
                 for (Object[] result : guestbookResults) {
                     StringBuilder sb = guestbookResponseService.convertGuestbookResponsesToCSV(customQandAs, datasetTitles, result);
                     os.write(sb.toString().getBytes());
-
                 }
             }
         };

From 6307292d3858bd62144e313de1b5574b55b4fb36 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 17 Oct 2023 13:07:14 -0400
Subject: [PATCH 577/922] more fixes/cleanup #9635

---
 .../search/SearchIncludeFragment.java         |  27 ++--
 .../dataverse/search/SearchServiceBean.java   | 127 ++++++++++--------
 2 files changed, 90 insertions(+), 64 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 1e42958fe4e..958ac0151c6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -308,15 +308,23 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             this.setRootDv(true);
         }
 
+        filterQueriesFinal.addAll(filterQueries);
+
+        /**
+         * Add type queries, for the types (Dataverses, Datasets, Datafiles) 
+         * currently selected:
+         */
         selectedTypesList = new ArrayList<>();
         String[] parts = selectedTypesString.split(":");
         selectedTypesList.addAll(Arrays.asList(parts));
-
-        
         
-        filterQueriesFinal.addAll(filterQueries);
-
+        logger.info("selected types list size: "+selectedTypesList.size());
         
+        String[] arr = selectedTypesList.toArray(new String[selectedTypesList.size()]);
+        selectedTypesHumanReadable = combine(arr, " OR ");
+        if (!selectedTypesHumanReadable.isEmpty()) {
+            typeFilterQuery = SearchFields.TYPE + ":(" + selectedTypesHumanReadable + ")";
+        }        
         filterQueriesFinal.add(typeFilterQuery);
 
         if (page <= 1) {
@@ -383,7 +391,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 // run another query to obtain the numbers of the unselected types:
                 
                 List<String> filterQueriesFinalSecondPass = new ArrayList<>();
-                filterQueriesFinalSecondPass.addAll(filterQueriesFinal);
+                filterQueriesFinalSecondPass.addAll(filterQueries);
                 
                 List<String> selectedTypesListSecondPass = new ArrayList<>();
                 
@@ -393,12 +401,13 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                     }
                 }
                 
-                String[] arr = selectedTypesListSecondPass.toArray(new String[selectedTypesListSecondPass.size()]);
+                arr = selectedTypesListSecondPass.toArray(new String[selectedTypesListSecondPass.size()]);
                 filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
-                
+
+                solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null, false, false);
+
                 if (solrQueryResponseSecondPass != null) {
 
-                    solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null);
                     if (solrQueryResponseSecondPass.hasError()) {
                         logger.info(solrQueryResponse.getError());
                         setSolrErrorEncountered(true);
@@ -410,6 +419,8 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                             previewCountbyType.put(facetLabel.getName(), facetLabel.getCount());
                         }
                     }
+                } else {
+                    logger.warning("null solr response from the second pass type query");
                 }
             }
             
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index aa2948eb8cb..d3ff7e42d15 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -195,64 +195,11 @@ public SolrQueryResponse search(
 //        }
 //        solrQuery.setSort(sortClause);
 
-        List<DatasetFieldType> datasetFields = datasetFieldService.findAllOrderedById();
-        Map<String, String> solrFieldsToHightlightOnMap = new HashMap<>();
-        if (addHighlights) {
-            solrQuery.setHighlight(true).setHighlightSnippets(1);
-            Integer fragSize = systemConfig.getSearchHighlightFragmentSize();
-            if (fragSize != null) {
-                solrQuery.setHighlightFragsize(fragSize);
-            }
-            solrQuery.setHighlightSimplePre("<span class=\"search-term-match\">");
-            solrQuery.setHighlightSimplePost("</span>");
-
-            // TODO: Do not hard code "Name" etc as English here.
-            solrFieldsToHightlightOnMap.put(SearchFields.NAME, "Name");
-            solrFieldsToHightlightOnMap.put(SearchFields.AFFILIATION, "Affiliation");
-            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_FRIENDLY, "File Type");
-            solrFieldsToHightlightOnMap.put(SearchFields.DESCRIPTION, "Description");
-            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NAME, "Variable Name");
-            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_LABEL, "Variable Label");
-            solrFieldsToHightlightOnMap.put(SearchFields.LITERAL_QUESTION, BundleUtil.getStringFromBundle("search.datasets.literalquestion"));
-            solrFieldsToHightlightOnMap.put(SearchFields.INTERVIEW_INSTRUCTIONS, BundleUtil.getStringFromBundle("search.datasets.interviewinstructions"));
-            solrFieldsToHightlightOnMap.put(SearchFields.POST_QUESTION, BundleUtil.getStringFromBundle("search.datasets.postquestion"));
-            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_UNIVERSE, BundleUtil.getStringFromBundle("search.datasets.variableuniverse"));
-            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NOTES, BundleUtil.getStringFromBundle("search.datasets.variableNotes"));
-
-            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_SEARCHABLE, "File Type");
-            solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PUBLICATION_DATE, "Publication Year");
-            solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.datasets.persistentId"));
-            solrFieldsToHightlightOnMap.put(SearchFields.FILE_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.files.persistentId"));
-            /**
-             * @todo Dataverse subject and affiliation should be highlighted but
-             * this is commented out right now because the "friendly" names are
-             * not being shown on the dataverse cards. See also
-             * https://github.com/IQSS/dataverse/issues/1431
-             */
-//        solrFieldsToHightlightOnMap.put(SearchFields.DATAVERSE_SUBJECT, "Subject");
-//        solrFieldsToHightlightOnMap.put(SearchFields.DATAVERSE_AFFILIATION, "Affiliation");
-            /**
-             * @todo: show highlight on file card?
-             * https://redmine.hmdc.harvard.edu/issues/3848
-             */
-            solrFieldsToHightlightOnMap.put(SearchFields.FILENAME_WITHOUT_EXTENSION, "Filename Without Extension");
-            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TAG_SEARCHABLE, "File Tag");
-
-            for (DatasetFieldType datasetFieldType : datasetFields) {
-                String solrField = datasetFieldType.getSolrField().getNameSearchable();
-                String displayName = datasetFieldType.getDisplayName();
-                solrFieldsToHightlightOnMap.put(solrField, displayName);
-            }
-            for (Map.Entry<String, String> entry : solrFieldsToHightlightOnMap.entrySet()) {
-                String solrField = entry.getKey();
-                // String displayName = entry.getValue();
-                solrQuery.addHighlightField(solrField);
-            }
-        }
         
         solrQuery.setParam("fl", "*,score");
         solrQuery.setParam("qt", "/select");
         solrQuery.setParam("facet", "true");
+        
         /**
          * @todo: do we need facet.query?
          */
@@ -315,7 +262,61 @@ public SolrQueryResponse search(
             }
         }
 
-        
+        List<DatasetFieldType> datasetFields = datasetFieldService.findAllOrderedById();
+        Map<String, String> solrFieldsToHightlightOnMap = new HashMap<>();
+        if (addHighlights) {
+            solrQuery.setHighlight(true).setHighlightSnippets(1);
+            Integer fragSize = systemConfig.getSearchHighlightFragmentSize();
+            if (fragSize != null) {
+                solrQuery.setHighlightFragsize(fragSize);
+            }
+            solrQuery.setHighlightSimplePre("<span class=\"search-term-match\">");
+            solrQuery.setHighlightSimplePost("</span>");
+
+            // TODO: Do not hard code "Name" etc as English here.
+            solrFieldsToHightlightOnMap.put(SearchFields.NAME, "Name");
+            solrFieldsToHightlightOnMap.put(SearchFields.AFFILIATION, "Affiliation");
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_FRIENDLY, "File Type");
+            solrFieldsToHightlightOnMap.put(SearchFields.DESCRIPTION, "Description");
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NAME, "Variable Name");
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_LABEL, "Variable Label");
+            solrFieldsToHightlightOnMap.put(SearchFields.LITERAL_QUESTION, BundleUtil.getStringFromBundle("search.datasets.literalquestion"));
+            solrFieldsToHightlightOnMap.put(SearchFields.INTERVIEW_INSTRUCTIONS, BundleUtil.getStringFromBundle("search.datasets.interviewinstructions"));
+            solrFieldsToHightlightOnMap.put(SearchFields.POST_QUESTION, BundleUtil.getStringFromBundle("search.datasets.postquestion"));
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_UNIVERSE, BundleUtil.getStringFromBundle("search.datasets.variableuniverse"));
+            solrFieldsToHightlightOnMap.put(SearchFields.VARIABLE_NOTES, BundleUtil.getStringFromBundle("search.datasets.variableNotes"));
+
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TYPE_SEARCHABLE, "File Type");
+            solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PUBLICATION_DATE, "Publication Year");
+            solrFieldsToHightlightOnMap.put(SearchFields.DATASET_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.datasets.persistentId"));
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_PERSISTENT_ID, BundleUtil.getStringFromBundle("advanced.search.files.persistentId"));
+            /**
+             * @todo Dataverse subject and affiliation should be highlighted but
+             * this is commented out right now because the "friendly" names are
+             * not being shown on the dataverse cards. See also
+             * https://github.com/IQSS/dataverse/issues/1431
+             */
+//        solrFieldsToHightlightOnMap.put(SearchFields.DATAVERSE_SUBJECT, "Subject");
+//        solrFieldsToHightlightOnMap.put(SearchFields.DATAVERSE_AFFILIATION, "Affiliation");
+            /**
+             * @todo: show highlight on file card?
+             * https://redmine.hmdc.harvard.edu/issues/3848
+             */
+            solrFieldsToHightlightOnMap.put(SearchFields.FILENAME_WITHOUT_EXTENSION, "Filename Without Extension");
+            solrFieldsToHightlightOnMap.put(SearchFields.FILE_TAG_SEARCHABLE, "File Tag");
+
+            for (DatasetFieldType datasetFieldType : datasetFields) {
+                String solrField = datasetFieldType.getSolrField().getNameSearchable();
+                String displayName = datasetFieldType.getDisplayName();
+                solrFieldsToHightlightOnMap.put(solrField, displayName);
+            }
+            for (Map.Entry<String, String> entry : solrFieldsToHightlightOnMap.entrySet()) {
+                String solrField = entry.getKey();
+                // String displayName = entry.getValue();
+                solrQuery.addHighlightField(solrField);
+            }
+        }
+
         //I'm not sure if just adding null here is good for hte permissions system... i think it needs something
         if(dataverses != null) {
             for(Dataverse dataverse : dataverses) {
@@ -370,7 +371,7 @@ public SolrQueryResponse search(
 //        solrQuery.addNumericRangeFacet(SearchFields.PRODUCTION_DATE_YEAR_ONLY, citationYearRangeStart, citationYearRangeEnd, citationYearRangeSpan);
 //        solrQuery.addNumericRangeFacet(SearchFields.DISTRIBUTION_DATE_YEAR_ONLY, citationYearRangeStart, citationYearRangeEnd, citationYearRangeSpan);
         solrQuery.setRows(numResultsPerPage);
-        logger.fine("Solr query:" + solrQuery);
+        logger.info("Solr query:" + solrQuery);
 
         // -----------------------------------
         // Make the solr query
@@ -378,8 +379,12 @@ public SolrQueryResponse search(
         QueryResponse queryResponse = null;
         try {
             queryResponse = solrClientService.getSolrClient().query(solrQuery);
+
         } catch (RemoteSolrException ex) {
             String messageFromSolr = ex.getLocalizedMessage();
+            
+            logger.info("message from solr exception: "+messageFromSolr);
+            
             String error = "Search Syntax Error: ";
             String stringToHide = "org.apache.solr.search.SyntaxError: ";
             if (messageFromSolr.startsWith(stringToHide)) {
@@ -393,6 +398,12 @@ public SolrQueryResponse search(
             exceptionSolrQueryResponse.setError(error);
 
             // we can't show anything because of the search syntax error
+            
+            // We probably shouldn't be assuming that this is necessarily a 
+            // "search syntax error" - could be anything else too - ? 
+            
+            
+            
             long zeroNumResultsFound = 0;
             long zeroGetResultsStart = 0;
             List<SolrSearchResult> emptySolrSearchResults = new ArrayList<>();
@@ -408,6 +419,10 @@ public SolrQueryResponse search(
         } catch (SolrServerException | IOException ex) {
             throw new SearchException("Internal Dataverse Search Engine Error", ex);
         }
+        
+        int statusCode = queryResponse.getStatus();
+        
+        logger.info("status code of the query response: "+statusCode);
 
         SolrDocumentList docs = queryResponse.getResults();
         List<SolrSearchResult> solrSearchResults = new ArrayList<>();

From 21eb153a63227fde85604b8d504c18813254496a Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Tue, 17 Oct 2023 20:47:00 +0200
Subject: [PATCH 578/922] Add API test for Dataverses GuestbookResponses

---
 .../iq/dataverse/api/DataversesIT.java        | 19 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 11 ++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 09052f9e4ea..171a35ac04f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -144,6 +144,25 @@ public void testMinimalDataverse() throws FileNotFoundException {
         deleteDataverse.then().assertThat().statusCode(OK.getStatusCode());
     }
 
+    @Test
+    public void testGetGuestbookResponses() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.prettyPrint();
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        // Create a Dataverse
+        Response create = UtilIT.createRandomDataverse(apiToken);
+        create.prettyPrint();
+        create.then().assertThat().statusCode(CREATED.getStatusCode());
+        String alias = UtilIT.getAliasFromResponse(create);
+        Integer dvId = UtilIT.getDataverseIdFromResponse(create);
+        // Get GuestbookResponses by Dataverse alias
+        Response getResponsesByAlias = UtilIT.getGuestbookResponses(alias, null, apiToken);
+        getResponsesByAlias.then().assertThat().statusCode(OK.getStatusCode());
+        // Get GuestbookResponses by Dataverse ID
+        Response getResponsesById = UtilIT.getGuestbookResponses(dvId.toString(), null, apiToken);
+        getResponsesById.then().assertThat().statusCode(OK.getStatusCode());
+    }
+
     @Test
     public void testNotEnoughJson() {
         Response createUser = UtilIT.createRandomUser();
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4421e9280b3..94a0f33a83e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -372,7 +372,16 @@ static Response showDataverseContents(String alias, String apiToken) {
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .when().get("/api/dataverses/" + alias + "/contents");
     }
-    
+
+    static Response getGuestbookResponses(String dataverseAlias, Long guestbookId, String apiToken) {
+        RequestSpecification requestSpec = given()
+                .auth().basic(apiToken, EMPTY_STRING);
+        if (guestbookId != null) {
+            requestSpec.queryParam("guestbookId", guestbookId);
+        }
+        return requestSpec.get("/api/dataverses/" + dataverseAlias + "/guestbookResponses/");
+    }
+
     static Response createRandomDatasetViaNativeApi(String dataverseAlias, String apiToken) {
         return createRandomDatasetViaNativeApi(dataverseAlias, apiToken, false);
     }

From 74eb7c551d209c9e460cbaea5572004b0fcad0bc Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 17 Oct 2023 16:09:32 -0400
Subject: [PATCH 579/922] more fixes (#9635)

---
 .../search/SearchIncludeFragment.java         | 24 +++++++++++++++----
 .../dataverse/search/SearchServiceBean.java   |  2 ++
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 958ac0151c6..177186fce49 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -395,9 +395,23 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 
                 List<String> selectedTypesListSecondPass = new ArrayList<>();
                 
-                for (String dvObjectType : previewCountbyType.keySet()) {
-                    if (previewCountbyType.get(dvObjectType) == -1) {
-                        selectedTypesListSecondPass.add(dvObjectType);
+                // @todo: simplify this!
+                for (String dvObjectTypeLabel : previewCountbyType.keySet()) {
+                    if (previewCountbyType.get(dvObjectTypeLabel) == -1) {
+                        String dvObjectType = null;
+                        
+                        if (dvObjectTypeLabel.equals(BundleUtil.getStringFromBundle("dataverses"))) {
+                            dvObjectType = "dataverses";
+                        } else if (dvObjectTypeLabel.equals(BundleUtil.getStringFromBundle("datasets"))) {
+                            dvObjectType = "datasets";
+                        } else if (dvObjectTypeLabel.equals(BundleUtil.getStringFromBundle("files"))) {
+                            dvObjectType = "files";
+                        }
+                    
+                        if (dvObjectType != null) {
+                            logger.info("adding object type to the second pass query: "+dvObjectType);
+                            selectedTypesListSecondPass.add(dvObjectType);
+                        }
                     }
                 }
                 
@@ -409,13 +423,15 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 if (solrQueryResponseSecondPass != null) {
 
                     if (solrQueryResponseSecondPass.hasError()) {
-                        logger.info(solrQueryResponse.getError());
+                        logger.info(solrQueryResponseSecondPass.getError());
                         setSolrErrorEncountered(true);
                     }
 
                     // And now populate the remaining type facets:
                     for (FacetCategory facetCategory : solrQueryResponseSecondPass.getTypeFacetCategories()) {
+                        logger.info("facet category: "+facetCategory.getName());
                         for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
+                            logger.info("facet label: "+facetLabel.getName());
                             previewCountbyType.put(facetLabel.getName(), facetLabel.getCount());
                         }
                     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index d3ff7e42d15..18cdbaa6994 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -423,6 +423,7 @@ public SolrQueryResponse search(
         int statusCode = queryResponse.getStatus();
         
         logger.info("status code of the query response: "+statusCode);
+        ///logger.info("number of hits: "+queryResponse._size());
 
         SolrDocumentList docs = queryResponse.getResults();
         List<SolrSearchResult> solrSearchResults = new ArrayList<>();
@@ -823,6 +824,7 @@ public SolrQueryResponse search(
             facetCategory.setFacetLabel(facetLabelList);
             if (!facetLabelList.isEmpty()) {
                 if (facetCategory.getName().equals(SearchFields.TYPE)) {
+                    logger.info("type facet encountered");
                     // the "type" facet is special, these are not
                     typeFacetCategories.add(facetCategory);
                 } else if (facetCategory.getName().equals(SearchFields.PUBLICATION_STATUS)) {

From f1e37ae0ff01e1fe0030202be1883f823bb8d080 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 17 Oct 2023 17:26:54 -0400
Subject: [PATCH 580/922] finally working as it should; much simplified/way
 less expensive second pass query sent in order to populate the unchecked type
 count facets. (#9635)

---
 .../iq/dataverse/search/SearchIncludeFragment.java  |  4 +---
 .../iq/dataverse/search/SearchServiceBean.java      | 13 +++++++++----
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 177186fce49..47a5621c3d6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -418,7 +418,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 arr = selectedTypesListSecondPass.toArray(new String[selectedTypesListSecondPass.size()]);
                 filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
 
-                solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null, false, false);
+                solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, null, sortOrder.toString(), 0, onlyDataRelatedToMe, 1, false, null, null, false, false);
 
                 if (solrQueryResponseSecondPass != null) {
 
@@ -429,9 +429,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
 
                     // And now populate the remaining type facets:
                     for (FacetCategory facetCategory : solrQueryResponseSecondPass.getTypeFacetCategories()) {
-                        logger.info("facet category: "+facetCategory.getName());
                         for (FacetLabel facetLabel : facetCategory.getFacetLabel()) {
-                            logger.info("facet label: "+facetLabel.getName());
                             previewCountbyType.put(facetLabel.getName(), facetLabel.getCount());
                         }
                     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 18cdbaa6994..be3330080c4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -165,7 +165,8 @@ public SolrQueryResponse search(
             List<Dataverse> dataverses,
             String query,
             List<String> filterQueries,
-            String sortField, String sortOrder,
+            String sortField, 
+            String sortOrder,
             int paginationStart,
             boolean onlyDatatRelatedToMe,
             int numResultsPerPage,
@@ -189,7 +190,11 @@ public SolrQueryResponse search(
 //        SortClause foo = new SortClause("name", SolrQuery.ORDER.desc);
 //        if (query.equals("*") || query.equals("*:*")) {
 //            solrQuery.setSort(new SortClause(SearchFields.NAME_SORT, SolrQuery.ORDER.asc));
-        solrQuery.setSort(new SortClause(sortField, sortOrder));
+        if (sortField != null) {
+            // is it ok not to specify any sort? - there are cases where we 
+            // don't care, and it must cost some extra cycles -- L.A.
+            solrQuery.setSort(new SortClause(sortField, sortOrder));
+        }
 //        } else {
 //            solrQuery.setSort(sortClause);
 //        }
@@ -423,7 +428,8 @@ public SolrQueryResponse search(
         int statusCode = queryResponse.getStatus();
         
         logger.info("status code of the query response: "+statusCode);
-        ///logger.info("number of hits: "+queryResponse._size());
+        logger.info("_size from query response: "+queryResponse._size());
+        logger.info("qtime: "+queryResponse.getQTime());
 
         SolrDocumentList docs = queryResponse.getResults();
         List<SolrSearchResult> solrSearchResults = new ArrayList<>();
@@ -824,7 +830,6 @@ public SolrQueryResponse search(
             facetCategory.setFacetLabel(facetLabelList);
             if (!facetLabelList.isEmpty()) {
                 if (facetCategory.getName().equals(SearchFields.TYPE)) {
-                    logger.info("type facet encountered");
                     // the "type" facet is special, these are not
                     typeFacetCategories.add(facetCategory);
                 } else if (facetCategory.getName().equals(SearchFields.PUBLICATION_STATUS)) {

From d0a858f5c817df7f626033063ec1afa4dbd69831 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 07:27:09 +0100
Subject: [PATCH 581/922] Added: ManageFilePermissions permission check to
 getUserPermissionsOnFile API endpoint

---
 .../harvard/iq/dataverse/FileDownloadServiceBean.java | 11 -----------
 .../java/edu/harvard/iq/dataverse/api/Access.java     |  3 ++-
 .../java/edu/harvard/iq/dataverse/api/AccessIT.java   |  2 ++
 3 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index de947ee9058..55817d4a746 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -645,15 +645,4 @@ public String getDirectStorageLocatrion(String storageLocation) {
             
         return null; 
     }
-
-    /**
-     *  Checks if the DataverseRequest, which contains IP Groups, has permission to download the file
-     *
-     * @param dataverseRequest the DataverseRequest
-     * @param dataFile the DataFile to check permissions
-     * @return boolean
-     */
-    public boolean canDownloadFile(DataverseRequest dataverseRequest, DataFile dataFile) {
-        return permissionService.requestOn(dataverseRequest, dataFile).has(Permission.DownloadFile);
-    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 1aaa7e60816..696fcb34920 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -1709,7 +1709,8 @@ public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @
         }
         JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
         User requestUser = getRequestUser(crc);
-        jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile));
+        jsonObjectBuilder.add("canDownloadFile", permissionService.userOn(requestUser, dataFile).has(Permission.DownloadFile));
+        jsonObjectBuilder.add("canManageFilePermissions", permissionService.userOn(requestUser, dataFile).has(Permission.ManageFilePermissions));
         jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset));
         return ok(jsonObjectBuilder);
     }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index 416caa68566..42e21e53101 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -666,6 +666,8 @@ public void testGetUserPermissionsOnFile() {
         assertTrue(canDownloadFile);
         boolean canEditOwnerDataset = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canEditOwnerDataset");
         assertTrue(canEditOwnerDataset);
+        boolean canManageFilePermissions = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canManageFilePermissions");
+        assertTrue(canManageFilePermissions);
 
         // Call with invalid file id
         Response getUserPermissionsOnFileInvalidIdResponse = UtilIT.getUserPermissionsOnFile("testInvalidId", apiToken);

From 5d8ac32754ea2c13c2dbd883d627b583a6cb1b43 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 07:34:58 +0100
Subject: [PATCH 582/922] Added: getUserPermissionsOnDataset API endpoint

---
 .../harvard/iq/dataverse/api/Datasets.java    | 20 +++++++++++
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 33 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 ++++
 3 files changed, 59 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index c3032495f27..7cfe587d8dc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -4083,4 +4083,24 @@ public Response resetGuestbookEntryAtRequest(@Context ContainerRequestContext cr
         datasetService.merge(dataset);
         return ok("Guestbook Entry At Request reset to default: " + dataset.getEffectiveGuestbookEntryAtRequest());
     }
+
+    @GET
+    @AuthRequired
+    @Path("{id}/userPermissions")
+    public Response getUserPermissionsOnDataset(@Context ContainerRequestContext crc, @PathParam("id") String datasetId) {
+        Dataset dataset;
+        try {
+            dataset = findDatasetOrDie(datasetId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        User requestUser = getRequestUser(crc);
+        JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder();
+        jsonObjectBuilder.add("canViewUnpublishedDataset", permissionService.userOn(requestUser, dataset).has(Permission.ViewUnpublishedDataset));
+        jsonObjectBuilder.add("canEditDataset", permissionService.userOn(requestUser, dataset).has(Permission.EditDataset));
+        jsonObjectBuilder.add("canPublishDataset", permissionService.userOn(requestUser, dataset).has(Permission.PublishDataset));
+        jsonObjectBuilder.add("canManageDatasetPermissions", permissionService.userOn(requestUser, dataset).has(Permission.ManageDatasetPermissions));
+        jsonObjectBuilder.add("canDeleteDatasetDraft", permissionService.userOn(requestUser, dataset).has(Permission.DeleteDatasetDraft));
+        return ok(jsonObjectBuilder);
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 34eccd3172a..4258773a0b3 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3928,4 +3928,37 @@ public void getDownloadSize() throws IOException, InterruptedException {
         getDownloadSizeResponse.then().assertThat().statusCode(OK.getStatusCode())
                 .body("data.storageSize", equalTo(expectedSizeIncludingAllSizes));
     }
+
+    @Test
+    public void testGetUserPermissionsOnDataset() {
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+
+        // Call with valid dataset id
+        Response getUserPermissionsOnDatasetResponse = UtilIT.getUserPermissionsOnDataset(Integer.toString(datasetId), apiToken);
+        getUserPermissionsOnDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+        boolean canViewUnpublishedDataset = JsonPath.from(getUserPermissionsOnDatasetResponse.body().asString()).getBoolean("data.canViewUnpublishedDataset");
+        assertTrue(canViewUnpublishedDataset);
+        boolean canEditDataset = JsonPath.from(getUserPermissionsOnDatasetResponse.body().asString()).getBoolean("data.canEditDataset");
+        assertTrue(canEditDataset);
+        boolean canPublishDataset = JsonPath.from(getUserPermissionsOnDatasetResponse.body().asString()).getBoolean("data.canPublishDataset");
+        assertTrue(canPublishDataset);
+        boolean canManageDatasetPermissions = JsonPath.from(getUserPermissionsOnDatasetResponse.body().asString()).getBoolean("data.canManageDatasetPermissions");
+        assertTrue(canManageDatasetPermissions);
+        boolean canDeleteDatasetDraft = JsonPath.from(getUserPermissionsOnDatasetResponse.body().asString()).getBoolean("data.canDeleteDatasetDraft");
+        assertTrue(canDeleteDatasetDraft);
+
+        // Call with invalid dataset id
+        Response getUserPermissionsOnDatasetInvalidIdResponse = UtilIT.getUserPermissionsOnDataset("testInvalidId", apiToken);
+        getUserPermissionsOnDatasetInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 4421e9280b3..be23df5ec63 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3359,6 +3359,12 @@ static Response getUserPermissionsOnFile(String dataFileId, String apiToken) {
                 .get("/api/access/datafile/" + dataFileId + "/userPermissions");
     }
 
+    static Response getUserPermissionsOnDataset(String datasetId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/datasets/" + datasetId + "/userPermissions");
+    }
+
     static Response createFileEmbargo(Integer datasetId, Integer fileId, String dateAvailable, String apiToken) {
         JsonObjectBuilder jsonBuilder = Json.createObjectBuilder();
         jsonBuilder.add("dateAvailable", dateAvailable);

From 38681bb113da3b9ea6359cf2da4e324e550ea463 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 07:40:24 +0100
Subject: [PATCH 583/922] Added: includeDeaccessioned optional query param to
 getVersion Datasets API endpoint

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 7cfe587d8dc..5e9d02c4af3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -482,9 +482,14 @@ public Response listVersions(@Context ContainerRequestContext crc, @PathParam("i
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}")
-    public Response getVersion(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+    public Response getVersion(@Context ContainerRequestContext crc,
+                               @PathParam("id") String datasetId,
+                               @PathParam("versionId") String versionId,
+                               @QueryParam("includeDeaccessioned") boolean includeDeaccessioned,
+                               @Context UriInfo uriInfo,
+                               @Context HttpHeaders headers) {
         return response( req -> {
-            DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers);
+            DatasetVersion dsv = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, includeDeaccessioned);
             return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
                     : ok(json(dsv));
         }, getRequestUser(crc));

From 835fb44325935a4509ce3139b96306b0370d290d Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 08:27:27 +0100
Subject: [PATCH 584/922] Added: docs for API endpoints
 getUserPermissionsOnDataset, getUserPermissionsOnFile and getVersion

---
 doc/sphinx-guides/source/api/dataaccess.rst |  1 +
 doc/sphinx-guides/source/api/native-api.rst | 30 +++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index 6edd413b7a5..f7aaa8f4ee4 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -426,6 +426,7 @@ This method returns the permissions that the calling user has on a particular fi
 In particular, the user permissions that this method checks, returned as booleans, are the following:
 
 * Can download the file
+* Can manage the file permissions
 * Can edit the file owner dataset
 
 A curl example using an ``id``::
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 3ac145b2f8e..f735079b334 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -909,6 +909,16 @@ The fully expanded example above (without environment variables) looks like this
 
   curl "https://demo.dataverse.org/api/datasets/24/versions/1.0"
 
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?includeDeaccessioned=true"
+
 .. _export-dataset-metadata-api:
 
 Export Metadata of a Dataset in Various Formats
@@ -2496,6 +2506,26 @@ The API can also be used to reset the dataset to use the default/inherited value
 
   curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
 
+Get User Permissions on a Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API call returns the permissions that the calling user has on a particular dataset.
+
+In particular, the user permissions that this method checks, returned as booleans, are the following:
+
+* Can view the unpublished dataset
+* Can edit the dataset
+* Can publish the dataset
+* Can manage the dataset permissions
+* Can delete the dataset draft
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=24
+
+  curl -H "X-Dataverse-key: $API_TOKEN" -X GET "$SERVER_URL/api/datasets/$ID/userPermissions"
 
 
 Files

From 56b291f4bf3a1ed4e48740ed50666a1709d4febf Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Wed, 18 Oct 2023 09:42:45 +0200
Subject: [PATCH 585/922] Log Dataverse ID instead of JSON path

---
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 94a0f33a83e..2f10e623047 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -287,7 +287,7 @@ static String getAliasFromResponse(Response createDataverseResponse) {
     static Integer getDataverseIdFromResponse(Response createDataverseResponse) {
         JsonPath createdDataverse = JsonPath.from(createDataverseResponse.body().asString());
         int dataverseId = createdDataverse.getInt("data.id");
-        logger.info("Id found in create dataverse response: " + createdDataverse);
+        logger.info("Id found in create dataverse response: " + dataverseId);
         return dataverseId;
     }
 

From e886c1adcd2cfe06e1b01a514350ba1f7f586cc1 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 09:04:27 +0100
Subject: [PATCH 586/922] Added: includeDeaccessioned IT test case for
 getDatasetVersion

---
 .../edu/harvard/iq/dataverse/api/DatasetsIT.java   | 14 +++++++++++++-
 .../java/edu/harvard/iq/dataverse/api/FilesIT.java |  8 ++++----
 .../java/edu/harvard/iq/dataverse/api/UtilIT.java  |  3 ++-
 3 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 4258773a0b3..569ebe0894b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -505,7 +505,7 @@ public void testCreatePublishDestroyDataset() {
         assertTrue(datasetContactFromExport.toString().contains("finch@mailinator.com"));
         assertTrue(firstValue.toString().contains("finch@mailinator.com"));
 
-        Response getDatasetVersion = UtilIT.getDatasetVersion(datasetPersistentId, DS_VERSION_LATEST_PUBLISHED, apiToken);
+        Response getDatasetVersion = UtilIT.getDatasetVersion(datasetPersistentId, DS_VERSION_LATEST_PUBLISHED, false, apiToken);
         getDatasetVersion.prettyPrint();
         getDatasetVersion.then().assertThat()
                 .body("data.datasetId", equalTo(datasetId))
@@ -549,6 +549,18 @@ public void testCreatePublishDestroyDataset() {
         }
         assertEquals(datasetPersistentId, XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.docDscr.citation.titlStmt.IDNo"));
 
+        // Test includeDeaccessioned option
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // includeDeaccessioned false
+        getDatasetVersion = UtilIT.getDatasetVersion(datasetPersistentId, DS_VERSION_LATEST_PUBLISHED, false, apiToken);
+        getDatasetVersion.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // includeDeaccessioned true
+        getDatasetVersion = UtilIT.getDatasetVersion(datasetPersistentId, DS_VERSION_LATEST_PUBLISHED, true, apiToken);
+        getDatasetVersion.then().assertThat().statusCode(OK.getStatusCode());
+
         Response deleteDatasetResponse = UtilIT.destroyDataset(datasetId, apiToken);
         deleteDatasetResponse.prettyPrint();
         assertEquals(200, deleteDatasetResponse.getStatusCode());
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 16726485dee..1f1321bad79 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -1989,14 +1989,14 @@ public void testDeleteFile() {
         deleteResponse2.then().assertThat().statusCode(OK.getStatusCode());
 
         // Check file 2 deleted from post v1.0 draft
-        Response postv1draft = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken);
+        Response postv1draft = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, false, apiToken);
         postv1draft.prettyPrint();
         postv1draft.then().assertThat()
                 .body("data.files.size()", equalTo(1))
                 .statusCode(OK.getStatusCode());
 
         // Check file 2 still in v1.0
-        Response v1 = UtilIT.getDatasetVersion(datasetPid, "1.0", apiToken);
+        Response v1 = UtilIT.getDatasetVersion(datasetPid, "1.0", false, apiToken);
         v1.prettyPrint();
         v1.then().assertThat()
                 .body("data.files[0].dataFile.filename", equalTo("cc0.png"))
@@ -2011,7 +2011,7 @@ public void testDeleteFile() {
         downloadResponse2.then().assertThat().statusCode(OK.getStatusCode());
 
         // Check file 3 still in post v1.0 draft
-        Response postv1draft2 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken);
+        Response postv1draft2 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, false, apiToken);
         postv1draft2.prettyPrint();
         postv1draft2.then().assertThat()
                 .body("data.files[0].dataFile.filename", equalTo("orcid_16x16.png"))
@@ -2026,7 +2026,7 @@ public void testDeleteFile() {
         deleteResponse3.then().assertThat().statusCode(OK.getStatusCode());
 
         // Check file 3 deleted from post v1.0 draft
-        Response postv1draft3 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, apiToken);
+        Response postv1draft3 = UtilIT.getDatasetVersion(datasetPid, DS_VERSION_DRAFT, false, apiToken);
         postv1draft3.prettyPrint();
         postv1draft3.then().assertThat()
                 .body("data.files[0]", equalTo(null))
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index be23df5ec63..0a1061c30ea 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -1399,9 +1399,10 @@ static Response nativeGetUsingPersistentId(String persistentId, String apiToken)
         return response;
     }
 
-    static Response getDatasetVersion(String persistentId, String versionNumber, String apiToken) {
+    static Response getDatasetVersion(String persistentId, String versionNumber, boolean includeDeaccessioned, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .queryParam("includeDeaccessioned", includeDeaccessioned)
                 .get("/api/datasets/:persistentId/versions/" + versionNumber + "?persistentId=" + persistentId);
     }
 

From 52d439d3284cf91064dbabcd3dbe401faeb3ba4d Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 09:10:53 +0100
Subject: [PATCH 587/922] Fixed: minor docs tweak

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index f735079b334..6f1c3072a55 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2511,7 +2511,7 @@ Get User Permissions on a Dataset
 
 This API call returns the permissions that the calling user has on a particular dataset.
 
-In particular, the user permissions that this method checks, returned as booleans, are the following:
+In particular, the user permissions that this API call checks, returned as booleans, are the following:
 
 * Can view the unpublished dataset
 * Can edit the dataset

From fa1b37bca410e903c9474ebf9aa6f38fd0b59c70 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 18 Oct 2023 09:12:37 +0100
Subject: [PATCH 588/922] Added: release notes for #10001

---
 .../10001-datasets-files-api-user-permissions.md    | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 doc/release-notes/10001-datasets-files-api-user-permissions.md

diff --git a/doc/release-notes/10001-datasets-files-api-user-permissions.md b/doc/release-notes/10001-datasets-files-api-user-permissions.md
new file mode 100644
index 00000000000..0aa75f9218a
--- /dev/null
+++ b/doc/release-notes/10001-datasets-files-api-user-permissions.md
@@ -0,0 +1,13 @@
+- New query parameter `includeDeaccessioned` added to the getVersion endpoint (/api/datasets/{id}/versions/{versionId}) to consider deaccessioned versions when searching for versions.
+  
+
+- New endpoint to get user permissions on a dataset (/api/datasets/{id}/userPermissions). In particular, the user permissions that this API call checks, returned as booleans, are the following:
+
+  - Can view the unpublished dataset
+  - Can edit the dataset
+  - Can publish the dataset
+  - Can manage the dataset permissions
+  - Can delete the dataset draft
+
+
+- New permission check "canManageFilePermissions" added to the existing endpoint for getting user permissions on a file (/api/access/datafile/{id}/userPermissions).
\ No newline at end of file

From 836d4a7006ea222f30f34d816c7e388a44d44142 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Wed, 18 Oct 2023 13:15:51 +0200
Subject: [PATCH 589/922] Authenticate using API token in header

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 2f10e623047..b4d77252615 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -375,7 +375,7 @@ static Response showDataverseContents(String alias, String apiToken) {
 
     static Response getGuestbookResponses(String dataverseAlias, Long guestbookId, String apiToken) {
         RequestSpecification requestSpec = given()
-                .auth().basic(apiToken, EMPTY_STRING);
+                .header(API_TOKEN_HTTP_HEADER, apiToken);
         if (guestbookId != null) {
             requestSpec.queryParam("guestbookId", guestbookId);
         }

From 7d7d73165e88c7791f1271bd55a3977134c978c9 Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Wed, 18 Oct 2023 13:20:23 +0200
Subject: [PATCH 590/922] Adjust expectations for getGuestbookResponses

---
 .../iq/dataverse/api/DataversesIT.java        | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
index 171a35ac04f..78ece6ecc42 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java
@@ -17,11 +17,13 @@
 import jakarta.json.Json;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
-import static jakarta.ws.rs.core.Response.Status.CREATED;
-import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
 import jakarta.ws.rs.core.Response.Status;
-import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 import static jakarta.ws.rs.core.Response.Status.OK;
+import static jakarta.ws.rs.core.Response.Status.CREATED;
+import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
+import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
+import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;
+import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
 
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -144,23 +146,42 @@ public void testMinimalDataverse() throws FileNotFoundException {
         deleteDataverse.then().assertThat().statusCode(OK.getStatusCode());
     }
 
+    /**
+     * A regular user can create a Dataverse Collection and access its
+     * GuestbookResponses by DV alias or ID.
+     * A request for a non-existent Dataverse's GuestbookResponses returns
+     * Not Found.
+     * A regular user cannot access the guestbook responses for a Dataverse
+     * that they do not have permissions for, like the root Dataverse.
+     */
     @Test
     public void testGetGuestbookResponses() {
         Response createUser = UtilIT.createRandomUser();
         createUser.prettyPrint();
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-        // Create a Dataverse
+
         Response create = UtilIT.createRandomDataverse(apiToken);
         create.prettyPrint();
         create.then().assertThat().statusCode(CREATED.getStatusCode());
         String alias = UtilIT.getAliasFromResponse(create);
         Integer dvId = UtilIT.getDataverseIdFromResponse(create);
-        // Get GuestbookResponses by Dataverse alias
+
+        logger.info("Request guestbook responses for non-existent Dataverse");
+        Response getResponsesByBadAlias = UtilIT.getGuestbookResponses("-1", null, apiToken);
+        getResponsesByBadAlias.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        logger.info("Request guestbook responses for existent Dataverse by alias");
         Response getResponsesByAlias = UtilIT.getGuestbookResponses(alias, null, apiToken);
         getResponsesByAlias.then().assertThat().statusCode(OK.getStatusCode());
-        // Get GuestbookResponses by Dataverse ID
+
+        logger.info("Request guestbook responses for existent Dataverse by ID");
         Response getResponsesById = UtilIT.getGuestbookResponses(dvId.toString(), null, apiToken);
         getResponsesById.then().assertThat().statusCode(OK.getStatusCode());
+
+        logger.info("Request guestbook responses for root Dataverse by alias");
+        getResponsesById = UtilIT.getGuestbookResponses("root", null, apiToken);
+        getResponsesById.prettyPrint();
+        getResponsesById.then().assertThat().statusCode(FORBIDDEN.getStatusCode());
     }
 
     @Test

From c1a19299e547fbc47322dafde74bc75d2e138d9c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 18 Oct 2023 13:48:47 -0400
Subject: [PATCH 591/922] a stub for interecepting a "circuit breaker" 503 from
 the server (#9635)

---
 .../dataverse/search/SearchServiceBean.java   | 31 ++++++++++++++-----
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index be3330080c4..1b92c2a4a46 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -382,13 +382,35 @@ public SolrQueryResponse search(
         // Make the solr query
         // -----------------------------------
         QueryResponse queryResponse = null;
+        boolean solrTemporarilyUnavailable = false; 
+        
         try {
             queryResponse = solrClientService.getSolrClient().query(solrQuery);
 
         } catch (RemoteSolrException ex) {
             String messageFromSolr = ex.getLocalizedMessage();
             
-            logger.info("message from solr exception: "+messageFromSolr);
+            logger.info("message from tye solr exception: "+messageFromSolr);
+            logger.info("code from the solr exception: "+ex.code());
+            
+            if (queryResponse != null) {
+                logger.info("return code: "+queryResponse.getStatus());
+            }
+            
+            // We probably shouldn't be assuming that this is necessarily a 
+            // "search syntax error", as the code below implies - could be 
+            // something else too - ? 
+            
+            // Specifically, we now rely on the Solr "circuit breaker" mechanism
+            // to start dropping requests with 503, when the service is 
+            // overwhelmed with requests load (with the assumption that this is
+            // a transient condition): 
+            
+            if (ex.code() == 503) {
+                solrTemporarilyUnavailable = true;
+                // actual logic for communicating this state back to the local 
+                // client code TBD (@todo)
+            }
             
             String error = "Search Syntax Error: ";
             String stringToHide = "org.apache.solr.search.SyntaxError: ";
@@ -403,12 +425,7 @@ public SolrQueryResponse search(
             exceptionSolrQueryResponse.setError(error);
 
             // we can't show anything because of the search syntax error
-            
-            // We probably shouldn't be assuming that this is necessarily a 
-            // "search syntax error" - could be anything else too - ? 
-            
-            
-            
+                        
             long zeroNumResultsFound = 0;
             long zeroGetResultsStart = 0;
             List<SolrSearchResult> emptySolrSearchResults = new ArrayList<>();

From 3c98c7d00e8e24be44e40b818d2ad2ff61a8ab29 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Thu, 19 Oct 2023 00:22:16 +0200
Subject: [PATCH 592/922] Fix identifer typo in docs

---
 .../source/developers/s3-direct-upload-api.rst            | 8 ++++----
 doc/sphinx-guides/source/user/dataset-management.rst      | 2 +-
 doc/sphinx-guides/source/user/find-use-data.rst           | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
index 4bf2bbdcc79..423fb02d385 100644
--- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
+++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
@@ -116,7 +116,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data
   curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
   
 Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
 
 To add multiple Uploaded Files to the Dataset
 ---------------------------------------------
@@ -147,7 +147,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data
   curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
 
 Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
 
 
 Replacing an existing file in the Dataset
@@ -177,7 +177,7 @@ Note that the API call does not validate that the file matches the hash value su
   curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/files/$FILE_IDENTIFIER/replace" -F "jsonData=$JSON_DATA"
   
 Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
 
 Replacing multiple existing files in the Dataset
 ------------------------------------------------
@@ -275,4 +275,4 @@ The JSON object returned as a response from this API call includes a "data" that
 
 
 Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst
index 1e8ea897032..bac0192bdd6 100755
--- a/doc/sphinx-guides/source/user/dataset-management.rst
+++ b/doc/sphinx-guides/source/user/dataset-management.rst
@@ -784,7 +784,7 @@ The "Compute" button on dataset and file pages will allow you to compute on a si
 Cloud Storage Access
 --------------------
 
-If you need to access a dataset in a more flexible way than the Compute button provides, then you can use the Cloud Storage Access box on the dataset page to copy the dataset's container name. This unique identifer can then be used to allow direct access to the dataset.
+If you need to access a dataset in a more flexible way than the Compute button provides, then you can use the Cloud Storage Access box on the dataset page to copy the dataset's container name. This unique identifier can then be used to allow direct access to the dataset.
 
 .. _deaccession:
 
diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst
index 2e82a1482b4..bea23cbcd0e 100755
--- a/doc/sphinx-guides/source/user/find-use-data.rst
+++ b/doc/sphinx-guides/source/user/find-use-data.rst
@@ -71,7 +71,7 @@ View Files
 
 Files in a Dataverse installation each have their own landing page that can be reached through the search results or through the Files table on their parent dataset's page. The dataset page and file page offer much the same functionality in terms of viewing and editing files, with a few small exceptions. 
 
-- In installations that have enabled support for persistent identifers (PIDs) at the file level, the file page includes the file's DOI or handle, which can be found in the file citation and also under the Metadata tab.
+- In installations that have enabled support for persistent identifiers (PIDs) at the file level, the file page includes the file's DOI or handle, which can be found in the file citation and also under the Metadata tab.
 - Previewers for several common file types are available and can be added by installation administrators.
 - The file page's Versions tab gives you a version history that is more focused on the individual file rather than the dataset as a whole. 
 

From d76e494f7c6889d9dfc23406e06b25f1a80d3507 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Thu, 19 Oct 2023 01:04:50 +0200
Subject: [PATCH 593/922] Fix identifer typo in code

Just in case code outside this project is referencing the old methods,
I created deprecated versions.
---
 .../dataverse/DatasetVersionServiceBean.java  | 26 ++++++++++++++-----
 .../java/edu/harvard/iq/dataverse/Shib.java   | 11 ++++----
 .../CollectionListManagerImpl.java            |  2 +-
 .../datadeposit/MediaResourceManagerImpl.java |  2 +-
 .../AuthenticationServiceBean.java            | 10 +++----
 .../authorization/UserIdentifier.java         | 21 ++++++++++++---
 .../providers/shib/ShibUtil.java              | 17 ++++++++++++
 .../filesystem/FileRecordJobListener.java     |  2 +-
 .../importer/filesystem/FileRecordReader.java |  2 +-
 .../impl/ImportFromFileSystemCommand.java     |  2 +-
 .../edu/harvard/iq/dataverse/api/AdminIT.java | 12 ++++-----
 .../providers/shib/ShibUtilTest.java          | 18 ++++++-------
 12 files changed, 85 insertions(+), 40 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index c2f9027a38a..cd3291e6222 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -495,10 +495,24 @@ private DatasetVersion getDatasetVersionByQuery(String queryString){
          }
     } // end getDatasetVersionByQuery
     
-    
-    
-    
-    public DatasetVersion retrieveDatasetVersionByIdentiferClause(String identifierClause, String version){
+    /**
+     * @deprecated because of a typo; use {@link #retrieveDatasetVersionByIdentifierClause(String, String) retrieveDatasetVersionByIdentifierClause} instead
+     * @see #retrieveDatasetVersionByIdentifierClause(String, String)
+     * @param identifierClause
+     * @param version
+     * @return a DatasetVersion if found, or {@code null} otherwise
+     */
+    @Deprecated
+    public DatasetVersion retrieveDatasetVersionByIdentiferClause(String identifierClause, String version) {
+        return retrieveDatasetVersionByIdentifierClause(identifierClause, version);
+    }
+
+    /**
+     * @param identifierClause
+     * @param version
+     * @return a DatasetVersion if found, or {@code null} otherwise
+     */
+    public DatasetVersion retrieveDatasetVersionByIdentifierClause(String identifierClause, String version) {
         
         if (identifierClause == null){
             return null;
@@ -620,7 +634,7 @@ public RetrieveDatasetVersionResponse retrieveDatasetVersionByPersistentId(Strin
         identifierClause += " AND ds.identifier = '" + parsedId.getIdentifier() + "'"; 
         
 
-        DatasetVersion ds = retrieveDatasetVersionByIdentiferClause(identifierClause, version);
+        DatasetVersion ds = retrieveDatasetVersionByIdentifierClause(identifierClause, version);
         
         if (ds != null){
             msg("retrieved dataset: " + ds.getId() + " semantic: " + ds.getSemanticVersion());
@@ -718,7 +732,7 @@ public DatasetVersion getDatasetVersionById(Long datasetId, String version){
         
         String identifierClause = this.getIdClause(datasetId);
 
-        DatasetVersion ds = retrieveDatasetVersionByIdentiferClause(identifierClause, version);
+        DatasetVersion ds = retrieveDatasetVersionByIdentifierClause(identifierClause, version);
         
         return ds;
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java
index bee1182e248..24c0f9d7926 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Shib.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java
@@ -24,6 +24,7 @@
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 import jakarta.ejb.EJB;
 import jakarta.ejb.EJBException;
@@ -62,7 +63,7 @@ public class Shib implements java.io.Serializable {
     HttpServletRequest request;
 
     private String userPersistentId;
-    private String internalUserIdentifer;
+    private String internalUserIdentifier;
     AuthenticatedUserDisplayInfo displayInfo;
     /**
      * @todo Remove this boolean some day? Now the mockups show a popup. Should
@@ -210,8 +211,8 @@ public void init() {
         }
 
         String usernameAssertion = getValueFromAssertion(ShibUtil.usernameAttribute);
-        internalUserIdentifer = ShibUtil.generateFriendlyLookingUserIdentifer(usernameAssertion, emailAddress);
-        logger.fine("friendly looking identifer (backend will enforce uniqueness):" + internalUserIdentifer);
+        internalUserIdentifier = ShibUtil.generateFriendlyLookingUserIdentifier(usernameAssertion, emailAddress);
+        logger.log(Level.FINE, "friendly looking identifier (backend will enforce uniqueness): {0}", internalUserIdentifier);
 
         String shibAffiliationAttribute = settingsService.getValueForKey(SettingsServiceBean.Key.ShibAffiliationAttribute);
         String affiliation = (StringUtils.isNotBlank(shibAffiliationAttribute))
@@ -326,7 +327,7 @@ public String confirmAndCreateAccount() {
         AuthenticatedUser au = null;
         try {
             au = authSvc.createAuthenticatedUser(
-                    new UserRecordIdentifier(shibAuthProvider.getId(), lookupStringPerAuthProvider), internalUserIdentifer, displayInfo, true);
+                    new UserRecordIdentifier(shibAuthProvider.getId(), lookupStringPerAuthProvider), internalUserIdentifier, displayInfo, true);
         } catch (EJBException ex) {
             /**
              * @todo Show the ConstraintViolationException, if any.
@@ -354,7 +355,7 @@ public String confirmAndConvertAccount() {
         visibleTermsOfUse = false;
         ShibAuthenticationProvider shibAuthProvider = new ShibAuthenticationProvider();
         String lookupStringPerAuthProvider = userPersistentId;
-        UserIdentifier userIdentifier = new UserIdentifier(lookupStringPerAuthProvider, internalUserIdentifer);
+        UserIdentifier userIdentifier = new UserIdentifier(lookupStringPerAuthProvider, internalUserIdentifier);
         logger.fine("builtin username: " + builtinUsername);
         AuthenticatedUser builtInUserToConvert = authSvc.canLogInAsBuiltinUser(builtinUsername, builtinPassword);
         if (builtInUserToConvert != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionListManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionListManagerImpl.java
index 084136f2b5d..541fa144e80 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionListManagerImpl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/CollectionListManagerImpl.java
@@ -94,7 +94,7 @@ public Feed listCollectionContents(IRI iri, AuthCredentials authCredentials, Swo
                 throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Could not find dataverse: " + dvAlias);
             }
         } else {
-            throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Couldn't determine target type or identifer from URL: " + iri);
+            throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Couldn't determine target type or identifier from URL: " + iri);
         }
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
index 15838a09456..a878720cc39 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
@@ -219,7 +219,7 @@ public void deleteMediaResource(String uri, AuthCredentials authCredentials, Swo
                 throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unsupported file type found in URL: " + uri);
             }
         } else {
-            throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Target or identifer not specified in URL: " + uri);
+            throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Target or identifier not specified in URL: " + uri);
         }
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
index 106a83a4ad1..496620cd6e8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
@@ -580,7 +580,7 @@ public boolean updateProvider( AuthenticatedUser authenticatedUser, String authe
      * {@code userDisplayInfo}, a lookup entry for them based
      * UserIdentifier.getLookupStringPerAuthProvider (within the supplied
      * authentication provider), and internal user identifier (used for role
-     * assignments, etc.) based on UserIdentifier.getInternalUserIdentifer.
+     * assignments, etc.) based on UserIdentifier.getInternalUserIdentifier.
      *
      * @param userRecordId
      * @param proposedAuthenticatedUserIdentifier
@@ -605,20 +605,20 @@ public AuthenticatedUser createAuthenticatedUser(UserRecordIdentifier userRecord
             proposedAuthenticatedUserIdentifier = proposedAuthenticatedUserIdentifier.trim();
         }
         // we now select a username for the generated AuthenticatedUser, or give up
-        String internalUserIdentifer = proposedAuthenticatedUserIdentifier;
+        String internalUserIdentifier = proposedAuthenticatedUserIdentifier;
         // TODO should lock table authenticated users for write here
-        if ( identifierExists(internalUserIdentifer) ) {
+        if ( identifierExists(internalUserIdentifier) ) {
             if ( ! generateUniqueIdentifier ) {
                 return null;
             }
             int i=1;
-            String identifier = internalUserIdentifer + i;
+            String identifier = internalUserIdentifier + i;
             while ( identifierExists(identifier) ) {
                 i += 1;
             }
             authenticatedUser.setUserIdentifier(identifier);
         } else {
-            authenticatedUser.setUserIdentifier(internalUserIdentifer);
+            authenticatedUser.setUserIdentifier(internalUserIdentifier);
         }
         authenticatedUser = save( authenticatedUser );
         // TODO should unlock table authenticated users for write here
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/UserIdentifier.java b/src/main/java/edu/harvard/iq/dataverse/authorization/UserIdentifier.java
index 1ac2c7583d6..312910e52c7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/UserIdentifier.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/UserIdentifier.java
@@ -25,18 +25,31 @@ public class UserIdentifier {
     /**
      * The String used in the permission system to assign roles, for example.
      */
-    String internalUserIdentifer;
+    String internalUserIdentifier;
 
-    public UserIdentifier(String lookupStringPerAuthProvider, String internalUserIdentifer) {
+    public UserIdentifier(String lookupStringPerAuthProvider, String internalUserIdentifier) {
         this.lookupStringPerAuthProvider = lookupStringPerAuthProvider;
-        this.internalUserIdentifer = internalUserIdentifer;
+        this.internalUserIdentifier = internalUserIdentifier;
     }
 
     public String getLookupStringPerAuthProvider() {
         return lookupStringPerAuthProvider;
     }
 
+    /**
+     * @deprecated because of a typo; use {@link #getInternalUserIdentifier()} instead
+     * @see #getInternalUserIdentifier()
+     * @return the internal user identifier
+     */
+    @Deprecated
     public String getInternalUserIdentifer() {
-        return internalUserIdentifer;
+        return getInternalUserIdentifier();
+    }
+
+    /**
+     * @return the internal user identifier
+     */
+    public String getInternalUserIdentifier() {
+        return internalUserIdentifier;
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
index fff135e0dec..4cf41903405 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtil.java
@@ -133,7 +133,24 @@ public static String findSingleValue(String mayHaveMultipleValues) {
         return singleValue;
     }
 
+    /**
+     * @deprecated because of a typo; use {@link #generateFriendlyLookingUserIdentifier(String, String)} instead
+     * @see #generateFriendlyLookingUserIdentifier(String, String)
+     * @param usernameAssertion
+     * @param email
+     * @return a friendly-looking user identifier based on the asserted username or email, or a UUID as fallback
+     */
+    @Deprecated
     public static String generateFriendlyLookingUserIdentifer(String usernameAssertion, String email) {
+        return generateFriendlyLookingUserIdentifier(usernameAssertion, email);
+    }
+
+    /**
+     * @param usernameAssertion
+     * @param email
+     * @return a friendly-looking user identifier based on the asserted username or email, or a UUID as fallback
+     */
+    public static String generateFriendlyLookingUserIdentifier(String usernameAssertion, String email) {
         if (usernameAssertion != null && !usernameAssertion.isEmpty()) {
             return usernameAssertion;
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java
index 593a5cbfdc3..a2f76150d7b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java
+++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordJobListener.java
@@ -450,7 +450,7 @@ private void loadChecksumManifest() {
         // We probably want package files to be able to use specific stores instead.
         // More importantly perhaps, the approach above does not take into account
         // if the dataset may have an AlternativePersistentIdentifier, that may be 
-        // designated isStorageLocationDesignator() - i.e., if a different identifer
+        // designated isStorageLocationDesignator() - i.e., if a different identifier
         // needs to be used to name the storage directory, instead of the main/current
         // persistent identifier above. 
         getJobLogger().log(Level.INFO, "Reading checksum manifest: " + manifestAbsolutePath);
diff --git a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordReader.java b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordReader.java
index fb702c21df2..9ce30683a87 100644
--- a/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordReader.java
+++ b/src/main/java/edu/harvard/iq/dataverse/batch/jobs/importer/filesystem/FileRecordReader.java
@@ -109,7 +109,7 @@ public void open(Serializable checkpoint) throws Exception {
         // We probably want package files to be able to use specific stores instead.
         // More importantly perhaps, the approach above does not take into account
         // if the dataset may have an AlternativePersistentIdentifier, that may be 
-        // designated isStorageLocationDesignator() - i.e., if a different identifer
+        // designated isStorageLocationDesignator() - i.e., if a different identifier
         // needs to be used to name the storage directory, instead of the main/current
         // persistent identifier above. 
         getJobLogger().log(Level.INFO, "Reading dataset directory: " + directory.getAbsolutePath() 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportFromFileSystemCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportFromFileSystemCommand.java
index c03c77d42fd..9a75f437b66 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportFromFileSystemCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ImportFromFileSystemCommand.java
@@ -83,7 +83,7 @@ public JsonObject execute(CommandContext ctxt) throws CommandException {
             //       We probably want package files to be able to use specific stores instead.
             //       More importantly perhaps, the approach above does not take into account
             //       if the dataset may have an AlternativePersistentIdentifier, that may be
-            //       designated isStorageLocationDesignator() - i.e., if a different identifer
+            //       designated isStorageLocationDesignator() - i.e., if a different identifier
             //       needs to be used to name the storage directory, instead of the main/current
             //       persistent identifier above.
             if (!isValidDirectory(directory)) {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
index a5a4924ad77..0c5de662e8a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
@@ -153,10 +153,10 @@ public void testFilterAuthenticatedUsers() throws Exception {
                 .body("data.pagination.pageCount", equalTo(1))
                 .body("data.pagination.numResults", equalTo(numResults));
         
-        String userIdentifer;
+        String userIdentifier;
         for (int i=0; i < numResults; i++){
-            userIdentifer = JsonPath.from(filterReponse01.getBody().asString()).getString("data.users[" + i + "].userIdentifier");
-            assertEquals(randomUsernames.contains(userIdentifer), true);
+            userIdentifier = JsonPath.from(filterReponse01.getBody().asString()).getString("data.users[" + i + "].userIdentifier");
+            assertTrue(randomUsernames.contains(userIdentifier));
         }
 
         List<Object> userList1 = JsonPath.from(filterReponse01.body().asString()).getList("data.users");
@@ -177,10 +177,10 @@ public void testFilterAuthenticatedUsers() throws Exception {
                 .body("data.pagination.pageCount", equalTo(3))
                 .body("data.pagination.numResults", equalTo(numResults));
         
-        String userIdentifer2;
+        String userIdentifier2;
         for (int i=0; i < numUsersReturned; i++){
-            userIdentifer2 = JsonPath.from(filterReponse02.getBody().asString()).getString("data.users[" + i + "].userIdentifier");
-            assertEquals(randomUsernames.contains(userIdentifer2), true);
+            userIdentifier2 = JsonPath.from(filterReponse02.getBody().asString()).getString("data.users[" + i + "].userIdentifier");
+            assertTrue(randomUsernames.contains(userIdentifier2));
         }
         
         List<Object> userList2 = JsonPath.from(filterReponse02.body().asString()).getList("data.users");
diff --git a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtilTest.java
index c644a4e2b2a..9ace90ac496 100644
--- a/src/test/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/authorization/providers/shib/ShibUtilTest.java
@@ -102,16 +102,16 @@ public void testFindSingleValue() {
         }
 
         @Test
-        public void testGenerateFriendlyLookingUserIdentifer() {
+        public void testGenerateFriendlyLookingUserIdentifier() {
             int lengthOfUuid = UUID.randomUUID().toString().length();
-            assertEquals("uid1", ShibUtil.generateFriendlyLookingUserIdentifer("uid1", null));
-            assertEquals(" leadingWhiteSpace", ShibUtil.generateFriendlyLookingUserIdentifer(" leadingWhiteSpace", null));
-            assertEquals("uid1", ShibUtil.generateFriendlyLookingUserIdentifer("uid1", "email1@example.com"));
-            assertEquals("email1", ShibUtil.generateFriendlyLookingUserIdentifer(null, "email1@example.com"));
-            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifer(null, null).length());
-            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifer(null, "").length());
-            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifer("", null).length());
-            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifer(null, "junkEmailAddress").length());
+            assertEquals("uid1", ShibUtil.generateFriendlyLookingUserIdentifier("uid1", null));
+            assertEquals(" leadingWhiteSpace", ShibUtil.generateFriendlyLookingUserIdentifier(" leadingWhiteSpace", null));
+            assertEquals("uid1", ShibUtil.generateFriendlyLookingUserIdentifier("uid1", "email1@example.com"));
+            assertEquals("email1", ShibUtil.generateFriendlyLookingUserIdentifier(null, "email1@example.com"));
+            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifier(null, null).length());
+            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifier(null, "").length());
+            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifier("", null).length());
+            assertEquals(lengthOfUuid, ShibUtil.generateFriendlyLookingUserIdentifier(null, "junkEmailAddress").length());
         }
 
         @Test

From 5bca73896f0b482a9c8f838d3a01d37d235b57ac Mon Sep 17 00:00:00 2001
From: Saikiran Patil <patilrsaikiran@gmail.com>
Date: Thu, 19 Oct 2023 18:41:02 +0530
Subject: [PATCH 594/922] Added tabulartags in files metadata for files
 metadata

---
 doc/sphinx-guides/source/api/native-api.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index eedf23fd04e..1e0804ce7d8 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3108,7 +3108,7 @@ A curl example using an ``ID``
   export ID=24
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false},"dataFileTags":["Survey"]}' \
     "$SERVER_URL/api/files/$ID/metadata"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3116,7 +3116,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false},"dataFileTags":["Survey"]}' \
     "http://demo.dataverse.org/api/files/24/metadata"
 
 A curl example using a ``PERSISTENT_ID``
@@ -3128,7 +3128,7 @@ A curl example using a ``PERSISTENT_ID``
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false},"dataFileTags":["Survey"]}' \
     "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3136,9 +3136,11 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}', "dataFileTags":["Survey"]} \
     "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
 
+Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.
+
 Also note that dataFileTags are not versioned and changes to these will update the published version of the file.
 
 .. _EditingVariableMetadata:

From ecbb020ed7da390c378fb76f08c9c5fb72677189 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 09:18:12 -0400
Subject: [PATCH 595/922] add/standardize retrieveSizeFromMedia call

---
 .../iq/dataverse/dataaccess/FileAccessIO.java |  33 +-
 .../dataverse/dataaccess/InputStreamIO.java   |   5 +
 .../dataaccess/RemoteOverlayAccessIO.java     |  14 +-
 .../iq/dataverse/dataaccess/S3AccessIO.java   |  21 +-
 .../iq/dataverse/dataaccess/StorageIO.java    | 379 +++++++++---------
 .../dataverse/dataaccess/SwiftAccessIO.java   |   5 +
 6 files changed, 241 insertions(+), 216 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
index d95df1567bd..3e6c802c526 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
@@ -35,8 +35,6 @@
 import java.util.List;
 import java.util.function.Predicate;
 import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 // Dataverse imports:
@@ -115,7 +113,7 @@ public void open (DataAccessOption... options) throws IOException {
 
                 this.setInputStream(fin);
                 setChannel(fin.getChannel());
-                this.setSize(getLocalFileSize());
+                this.setSize(retrieveSizeFromMedia());
 
                 if (dataFile.getContentType() != null
                         && dataFile.getContentType().equals("text/tab-separated-values")
@@ -506,21 +504,6 @@ public void delete() throws IOException {
     
     // Auxilary helper methods, filesystem access-specific:
     
-    private long getLocalFileSize () {
-        long fileSize = -1;
-
-        try {
-            File testFile = getFileSystemPath().toFile();
-            if (testFile != null) {
-                fileSize = testFile.length();
-            }
-            return fileSize;
-        } catch (IOException ex) {
-            return -1;
-        }
-
-    }
-
     public FileInputStream openLocalFileAsInputStream () {
         FileInputStream in;
 
@@ -742,4 +725,18 @@ public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOE
         return toDelete;
     }
 
+    @Override
+    public long retrieveSizeFromMedia() {
+        long fileSize = -1;
+        try {
+            File testFile = getFileSystemPath().toFile();
+            if (testFile != null) {
+                fileSize = testFile.length();
+            }
+            return fileSize;
+        } catch (IOException ex) {
+            return -1;
+        }
+    }
+
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/InputStreamIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/InputStreamIO.java
index be6f9df0254..de392b74cca 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/InputStreamIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/InputStreamIO.java
@@ -165,4 +165,9 @@ public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOE
         throw new UnsupportedDataAccessOperationException("InputStreamIO: tthis method is not supported in this DataAccess driver.");
     }
 
+    @Override
+    public long retrieveSizeFromMedia() throws UnsupportedDataAccessOperationException {
+        throw new UnsupportedDataAccessOperationException("InputStreamIO: this method is not supported in this DataAccess driver.");
+    }
+
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index a9653f2ab68..9c1f5ba23aa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -64,8 +64,6 @@
 public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
-
-    String globusAccessToken = null;
     
     protected StorageIO<DvObject> baseStore = null;
     protected String path = null;
@@ -155,7 +153,7 @@ public void open(DataAccessOption... options) throws IOException {
                     this.setSize(dataFile.getFilesize());
                 } else {
                     logger.fine("Setting size");
-                    this.setSize(retrieveSize());
+                    this.setSize(retrieveSizeFromMedia());
                 }
                 if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values")
                         && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
@@ -183,7 +181,8 @@ public void open(DataAccessOption... options) throws IOException {
         }
     }
 
-    long retrieveSize() {
+    @Override
+    public long retrieveSizeFromMedia() {
         long size = -1;
         HttpHead head = new HttpHead(baseUrl + "/" + path);
         try {
@@ -383,7 +382,7 @@ public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
     @Override
     public boolean exists() {
         logger.fine("Exists called");
-        return (retrieveSize() != -1);
+        return (retrieveSizeFromMedia() != -1);
     }
 
     @Override
@@ -502,8 +501,9 @@ protected void configureStores(DataAccessRequest req, String driverId, String st
                     if (index > 0) {
                         storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
                     }
-                    // THe base store needs the baseStoreIdentifier and not the relative URL
-                    fullStorageLocation = storageLocation.substring(0, storageLocation.indexOf("//"));
+                    // The base store needs the baseStoreIdentifier and not the relative URL (if it exists)
+                    int endOfId = storageLocation.indexOf("//");
+                    fullStorageLocation = (endOfId>-1) ? storageLocation.substring(0, endOfId) : storageLocation;
 
                     switch (baseDriverType) {
                     case DataAccess.S3:
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index 822ada0b83e..b0f9f0ffb05 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -207,14 +207,7 @@ public void open(DataAccessOption... options) throws IOException {
 
             
             if (isReadAccess) {
-                key = getMainFileKey();
-                ObjectMetadata objectMetadata = null; 
-                try {
-                    objectMetadata = s3.getObjectMetadata(bucketName, key);
-                } catch (SdkClientException sce) {
-                    throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")");
-                }
-                this.setSize(objectMetadata.getContentLength());
+                this.setSize(retrieveSizeFromMedia());
 
                 if (dataFile.getContentType() != null
                         && dataFile.getContentType().equals("text/tab-separated-values")
@@ -1385,4 +1378,16 @@ public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOE
         }
         return toDelete;
     }
+
+    @Override
+    public long retrieveSizeFromMedia() throws IOException {
+        key = getMainFileKey();
+        ObjectMetadata objectMetadata = null;
+        try {
+            objectMetadata = s3.getObjectMetadata(bucketName, key);
+        } catch (SdkClientException sce) {
+            throw new IOException("Cannot get S3 object " + key + " (" + sce.getMessage() + ")");
+        }
+        return objectMetadata.getContentLength();
+    }
 }
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index 3bc83538679..f3c2ef5f513 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -20,7 +20,6 @@
 
 package edu.harvard.iq.dataverse.dataaccess;
 
-
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.Dataverse;
@@ -43,7 +42,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-
 /**
  *
  * @author Leonid Andreev
@@ -55,15 +53,15 @@ public abstract class StorageIO<T extends DvObject> {
     public StorageIO() {
 
     }
-    
+
     public StorageIO(String storageLocation, String driverId) {
-      this.driverId=driverId;
+        this.driverId = driverId;
     }
 
     public StorageIO(T dvObject, DataAccessRequest req, String driverId) {
         this.dvObject = dvObject;
         this.req = req;
-        this.driverId=driverId;
+        this.driverId = driverId;
         if (this.req == null) {
             this.req = new DataAccessRequest();
         }
@@ -72,18 +70,19 @@ public StorageIO(T dvObject, DataAccessRequest req, String driverId) {
         }
     }
 
-    
-    
     // Abstract methods to be implemented by the storage drivers:
 
     public abstract void open(DataAccessOption... option) throws IOException;
 
     protected boolean isReadAccess = false;
     protected boolean isWriteAccess = false;
-    //A  public store is one in which files may be accessible outside Dataverse and therefore accessible without regard to Dataverse's access controls related to restriction and embargoes.
-    //Currently, this is just used to warn users at upload time rather than disable restriction/embargo. 
+    // A public store is one in which files may be accessible outside Dataverse and
+    // therefore accessible without regard to Dataverse's access controls related to
+    // restriction and embargoes.
+    // Currently, this is just used to warn users at upload time rather than disable
+    // restriction/embargo.
     static protected Map<String, Boolean> driverPublicAccessMap = new HashMap<String, Boolean>();
-    
+
     public boolean canRead() {
         return isReadAccess;
     }
@@ -94,115 +93,118 @@ public boolean canWrite() {
 
     public abstract String getStorageLocation() throws IOException;
 
-    // This method will return a Path, if the storage method is a 
-    // local filesystem. Otherwise should throw an IOException. 
+    // This method will return a Path, if the storage method is a
+    // local filesystem. Otherwise should throw an IOException.
     public abstract Path getFileSystemPath() throws IOException;
-        
-    public abstract boolean exists() throws IOException; 
-        
+
+    public abstract boolean exists() throws IOException;
+
     public abstract void delete() throws IOException;
-    
+
     // this method for copies a local Path (for ex., a
     // temp file, into this DataAccess location):
     public abstract void savePath(Path fileSystemPath) throws IOException;
-    
+
     // same, for an InputStream:
     /**
-     * This method copies a local InputStream into this DataAccess location.
-     * Note that the S3 driver implementation of this abstract method is problematic, 
-     * because S3 cannot save an object of an unknown length. This effectively 
-     * nullifies any benefits of streaming; as we cannot start saving until we 
-     * have read the entire stream. 
-     * One way of solving this would be to buffer the entire stream as byte[], 
-     * in memory, then save it... Which of course would be limited by the amount 
-     * of memory available, and thus would not work for streams larger than that. 
-     * So we have eventually decided to save save the stream to a temp file, then 
-     * save to S3. This is slower, but guaranteed to work on any size stream. 
-     * An alternative we may want to consider is to not implement this method 
-     * in the S3 driver, and make it throw the UnsupportedDataAccessOperationException, 
-     * similarly to how we handle attempts to open OutputStreams, in this and the 
-     * Swift driver. 
-     * (Not an issue in either FileAccessIO or SwiftAccessIO implementations)
+     * This method copies a local InputStream into this DataAccess location. Note
+     * that the S3 driver implementation of this abstract method is problematic,
+     * because S3 cannot save an object of an unknown length. This effectively
+     * nullifies any benefits of streaming; as we cannot start saving until we have
+     * read the entire stream. One way of solving this would be to buffer the entire
+     * stream as byte[], in memory, then save it... Which of course would be limited
+     * by the amount of memory available, and thus would not work for streams larger
+     * than that. So we have eventually decided to save save the stream to a temp
+     * file, then save to S3. This is slower, but guaranteed to work on any size
+     * stream. An alternative we may want to consider is to not implement this
+     * method in the S3 driver, and make it throw the
+     * UnsupportedDataAccessOperationException, similarly to how we handle attempts
+     * to open OutputStreams, in this and the Swift driver. (Not an issue in either
+     * FileAccessIO or SwiftAccessIO implementations)
      * 
      * @param inputStream InputStream we want to save
-     * @param auxItemTag String representing this Auxiliary type ("extension")
+     * @param auxItemTag  String representing this Auxiliary type ("extension")
      * @throws IOException if anything goes wrong.
-    */
+     */
     public abstract void saveInputStream(InputStream inputStream) throws IOException;
+
     public abstract void saveInputStream(InputStream inputStream, Long filesize) throws IOException;
-    
+
     // Auxiliary File Management: (new as of 4.0.2!)
-    
+
     // An "auxiliary object" is an abstraction of the traditional DVN/Dataverse
-    // mechanism of storing extra files related to the man StudyFile/DataFile - 
-    // such as "saved original" and cached format conversions for tabular files, 
-    // thumbnails for images, etc. - in physical files with the same file 
-    // name but various reserved extensions. 
-   
-    //This function retrieves auxiliary files related to datasets, and returns them as inputstream
-    public abstract InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException ;
-    
+    // mechanism of storing extra files related to the man StudyFile/DataFile -
+    // such as "saved original" and cached format conversions for tabular files,
+    // thumbnails for images, etc. - in physical files with the same file
+    // name but various reserved extensions.
+
+    // This function retrieves auxiliary files related to datasets, and returns them
+    // as inputstream
+    public abstract InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException;
+
     public abstract Channel openAuxChannel(String auxItemTag, DataAccessOption... option) throws IOException;
-    
-    public abstract long getAuxObjectSize(String auxItemTag) throws IOException; 
-    
-    public abstract Path getAuxObjectAsPath(String auxItemTag) throws IOException; 
-    
-    public abstract boolean isAuxObjectCached(String auxItemTag) throws IOException; 
-    
-    public abstract void backupAsAux(String auxItemTag) throws IOException; 
-    
-    public abstract void revertBackupAsAux(String auxItemTag) throws IOException; 
-    
-    // this method copies a local filesystem Path into this DataAccess Auxiliary location:
+
+    public abstract long getAuxObjectSize(String auxItemTag) throws IOException;
+
+    public abstract Path getAuxObjectAsPath(String auxItemTag) throws IOException;
+
+    public abstract boolean isAuxObjectCached(String auxItemTag) throws IOException;
+
+    public abstract void backupAsAux(String auxItemTag) throws IOException;
+
+    public abstract void revertBackupAsAux(String auxItemTag) throws IOException;
+
+    // this method copies a local filesystem Path into this DataAccess Auxiliary
+    // location:
     public abstract void savePathAsAux(Path fileSystemPath, String auxItemTag) throws IOException;
-    
+
     /**
-     * This method copies a local InputStream into this DataAccess Auxiliary location.
-     * Note that the S3 driver implementation of this abstract method is problematic, 
-     * because S3 cannot save an object of an unknown length. This effectively 
-     * nullifies any benefits of streaming; as we cannot start saving until we 
-     * have read the entire stream. 
-     * One way of solving this would be to buffer the entire stream as byte[], 
-     * in memory, then save it... Which of course would be limited by the amount 
-     * of memory available, and thus would not work for streams larger than that. 
-     * So we have eventually decided to save save the stream to a temp file, then 
-     * save to S3. This is slower, but guaranteed to work on any size stream. 
-     * An alternative we may want to consider is to not implement this method 
-     * in the S3 driver, and make it throw the UnsupportedDataAccessOperationException, 
-     * similarly to how we handle attempts to open OutputStreams, in this and the 
-     * Swift driver. 
-     * (Not an issue in either FileAccessIO or SwiftAccessIO implementations)
+     * This method copies a local InputStream into this DataAccess Auxiliary
+     * location. Note that the S3 driver implementation of this abstract method is
+     * problematic, because S3 cannot save an object of an unknown length. This
+     * effectively nullifies any benefits of streaming; as we cannot start saving
+     * until we have read the entire stream. One way of solving this would be to
+     * buffer the entire stream as byte[], in memory, then save it... Which of
+     * course would be limited by the amount of memory available, and thus would not
+     * work for streams larger than that. So we have eventually decided to save save
+     * the stream to a temp file, then save to S3. This is slower, but guaranteed to
+     * work on any size stream. An alternative we may want to consider is to not
+     * implement this method in the S3 driver, and make it throw the
+     * UnsupportedDataAccessOperationException, similarly to how we handle attempts
+     * to open OutputStreams, in this and the Swift driver. (Not an issue in either
+     * FileAccessIO or SwiftAccessIO implementations)
      * 
      * @param inputStream InputStream we want to save
-     * @param auxItemTag String representing this Auxiliary type ("extension")
+     * @param auxItemTag  String representing this Auxiliary type ("extension")
      * @throws IOException if anything goes wrong.
-    */
-    public abstract void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException; 
-    public abstract void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Long filesize) throws IOException;
-    
-    public abstract List<String>listAuxObjects() throws IOException;
-    
-    public abstract void deleteAuxObject(String auxItemTag) throws IOException; 
-    
+     */
+    public abstract void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException;
+
+    public abstract void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Long filesize)
+            throws IOException;
+
+    public abstract List<String> listAuxObjects() throws IOException;
+
+    public abstract void deleteAuxObject(String auxItemTag) throws IOException;
+
     public abstract void deleteAllAuxObjects() throws IOException;
 
     private DataAccessRequest req;
     private InputStream in = null;
-    private OutputStream out; 
+    private OutputStream out;
     protected Channel channel;
     protected DvObject dvObject;
     protected String driverId;
 
-    /*private int status;*/
+    /* private int status; */
     private long size;
 
     /**
-     * Where in the file to seek to when reading (default is zero bytes, the
-     * start of the file).
+     * Where in the file to seek to when reading (default is zero bytes, the start
+     * of the file).
      */
     private long offset;
-    
+
     private String mimeType;
     private String fileName;
     private String varHeader;
@@ -215,8 +217,8 @@ public boolean canWrite() {
     private String swiftContainerName;
 
     private boolean isLocalFile = false;
-    /*private boolean isRemoteAccess = false;*/
-    /*private boolean isHttpAccess = false;*/
+    /* private boolean isRemoteAccess = false; */
+    /* private boolean isHttpAccess = false; */
     private boolean noVarHeader = false;
 
     // For remote downloads:
@@ -229,13 +231,14 @@ public boolean canWrite() {
     private String remoteUrl;
     protected String remoteStoreName = null;
     protected URL remoteStoreUrl = null;
-    
+
     // For HTTP-based downloads:
-    /*private GetMethod method = null;
-    private Header[] responseHeaders;*/
+    /*
+     * private GetMethod method = null; private Header[] responseHeaders;
+     */
 
     // getters:
-    
+
     public Channel getChannel() throws IOException {
         return channel;
     }
@@ -255,16 +258,15 @@ public ReadableByteChannel getReadChannel() throws IOException {
 
         return (ReadableByteChannel) channel;
     }
-    
-    public DvObject getDvObject()
-    {
+
+    public DvObject getDvObject() {
         return dvObject;
     }
-    
+
     public DataFile getDataFile() {
         return (DataFile) dvObject;
     }
-    
+
     public Dataset getDataset() {
         return (Dataset) dvObject;
     }
@@ -277,9 +279,9 @@ public DataAccessRequest getRequest() {
         return req;
     }
 
-    /*public int getStatus() {
-        return status;
-    }*/
+    /*
+     * public int getStatus() { return status; }
+     */
 
     public long getSize() {
         return size;
@@ -292,9 +294,9 @@ public long getOffset() {
     public InputStream getInputStream() throws IOException {
         return in;
     }
-    
+
     public OutputStream getOutputStream() throws IOException {
-        return out; 
+        return out;
     }
 
     public String getMimeType() {
@@ -317,23 +319,23 @@ public String getRemoteUrl() {
         return remoteUrl;
     }
 
-    public String getTemporarySwiftUrl(){
+    public String getTemporarySwiftUrl() {
         return temporarySwiftUrl;
     }
-    
+
     public String getTempUrlExpiry() {
         return tempUrlExpiry;
     }
-    
+
     public String getTempUrlSignature() {
         return tempUrlSignature;
     }
-    
+
     public String getSwiftFileName() {
         return swiftFileName;
     }
 
-    public String getSwiftContainerName(){
+    public String getSwiftContainerName() {
         return swiftContainerName;
     }
 
@@ -344,34 +346,32 @@ public String getRemoteStoreName() {
     public URL getRemoteStoreUrl() {
         return remoteStoreUrl;
     }
-    
-    /*public GetMethod getHTTPMethod() {
-        return method;
-    }
 
-    public Header[] getResponseHeaders() {
-        return responseHeaders;
-    }*/
+    /*
+     * public GetMethod getHTTPMethod() { return method; }
+     * 
+     * public Header[] getResponseHeaders() { return responseHeaders; }
+     */
 
     public boolean isLocalFile() {
         return isLocalFile;
     }
-    
-    // "Direct Access" StorageIO is used to access a physical storage 
-    // location not associated with any dvObject. (For example, when we 
-    // are deleting a physical file left behind by a DataFile that's 
-    // already been deleted from the database). 
+
+    // "Direct Access" StorageIO is used to access a physical storage
+    // location not associated with any dvObject. (For example, when we
+    // are deleting a physical file left behind by a DataFile that's
+    // already been deleted from the database).
     public boolean isDirectAccess() {
-        return dvObject == null; 
+        return dvObject == null;
     }
 
-    /*public boolean isRemoteAccess() {
-        return isRemoteAccess;
-    }*/
+    /*
+     * public boolean isRemoteAccess() { return isRemoteAccess; }
+     */
 
-    /*public boolean isHttpAccess() {
-        return isHttpAccess;
-    }*/
+    /*
+     * public boolean isHttpAccess() { return isHttpAccess; }
+     */
 
     public boolean isDownloadSupported() {
         return isDownloadSupported;
@@ -398,9 +398,9 @@ public void setRequest(DataAccessRequest dar) {
         req = dar;
     }
 
-    /*public void setStatus(int s) {
-        status = s;
-    }*/
+    /*
+     * public void setStatus(int s) { status = s; }
+     */
 
     public void setSize(long s) {
         size = s;
@@ -421,11 +421,11 @@ public void setOffset(long offset) throws IOException {
     public void setInputStream(InputStream is) {
         in = is;
     }
-    
+
     public void setOutputStream(OutputStream os) {
-        out = os; 
-    } 
-    
+        out = os;
+    }
+
     public void setChannel(Channel c) {
         channel = c;
     }
@@ -450,45 +450,46 @@ public void setRemoteUrl(String u) {
         remoteUrl = u;
     }
 
-    public void setTemporarySwiftUrl(String u){
+    public void setTemporarySwiftUrl(String u) {
         temporarySwiftUrl = u;
     }
-    
-    public void setTempUrlExpiry(Long u){
+
+    public void setTempUrlExpiry(Long u) {
         tempUrlExpiry = String.valueOf(u);
     }
-    
+
     public void setSwiftFileName(String u) {
         swiftFileName = u;
     }
-    
-    public void setTempUrlSignature(String u){
+
+    public void setTempUrlSignature(String u) {
         tempUrlSignature = u;
     }
 
-    public void setSwiftContainerName(String u){
+    public void setSwiftContainerName(String u) {
         swiftContainerName = u;
     }
 
-    /*public void setHTTPMethod(GetMethod hm) {
-        method = hm;
-    }*/
+    /*
+     * public void setHTTPMethod(GetMethod hm) { method = hm; }
+     */
 
-    /*public void setResponseHeaders(Header[] headers) {
-        responseHeaders = headers;
-    }*/
+    /*
+     * public void setResponseHeaders(Header[] headers) { responseHeaders = headers;
+     * }
+     */
 
     public void setIsLocalFile(boolean f) {
         isLocalFile = f;
     }
 
-    /*public void setIsRemoteAccess(boolean r) {
-        isRemoteAccess = r;
-    }*/
+    /*
+     * public void setIsRemoteAccess(boolean r) { isRemoteAccess = r; }
+     */
 
-    /*public void setIsHttpAccess(boolean h) {
-        isHttpAccess = h;
-    }*/
+    /*
+     * public void setIsHttpAccess(boolean h) { isHttpAccess = h; }
+     */
 
     public void setIsDownloadSupported(boolean d) {
         isDownloadSupported = d;
@@ -506,12 +507,11 @@ public void setNoVarHeader(boolean nvh) {
         noVarHeader = nvh;
     }
 
-        // connection management methods:
-    /*public void releaseConnection() {
-        if (method != null) {
-            method.releaseConnection();
-        }
-    }*/
+    // connection management methods:
+    /*
+     * public void releaseConnection() { if (method != null) {
+     * method.releaseConnection(); } }
+     */
 
     public void closeInputStream() {
         if (in != null) {
@@ -528,7 +528,7 @@ public void closeInputStream() {
             }
         }
     }
-    
+
     public String generateVariableHeader(List<DataVariable> dvs) {
         String varHeader = null;
 
@@ -571,14 +571,14 @@ protected boolean isWriteAccessRequested(DataAccessOption... options) throws IOE
         return false;
     }
 
-	public boolean isBelowIngestSizeLimit() {
-		long limit = Long.parseLong(System.getProperty("dataverse.files." + this.driverId + ".ingestsizelimit", "-1"));
-		if(limit>0 && getSize()>limit) {
-			return false;
-		} else {
-		    return true;
-		}
-	}
+    public boolean isBelowIngestSizeLimit() {
+        long limit = Long.parseLong(System.getProperty("dataverse.files." + this.driverId + ".ingestsizelimit", "-1"));
+        if (limit > 0 && getSize() > limit) {
+            return false;
+        } else {
+            return true;
+        }
+    }
 
     public boolean downloadRedirectEnabled() {
         return false;
@@ -587,36 +587,38 @@ public boolean downloadRedirectEnabled() {
     public boolean downloadRedirectEnabled(String auxObjectTag) {
         return false;
     }
-    
-    public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName) throws IOException {
+
+    public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName)
+            throws IOException {
         throw new UnsupportedDataAccessOperationException("Direct download not implemented for this storage type");
     }
-    
 
     public static boolean isPublicStore(String driverId) {
-        //Read once and cache
-        if(!driverPublicAccessMap.containsKey(driverId)) {
-            driverPublicAccessMap.put(driverId, Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".public")));
+        // Read once and cache
+        if (!driverPublicAccessMap.containsKey(driverId)) {
+            driverPublicAccessMap.put(driverId,
+                    Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".public")));
         }
         return driverPublicAccessMap.get(driverId);
     }
-    
+
     public static String getDriverPrefix(String driverId) {
-        return driverId+ DataAccess.SEPARATOR;
+        return driverId + DataAccess.SEPARATOR;
     }
-    
+
     public static boolean isDirectUploadEnabled(String driverId) {
-        return (System.getProperty("dataverse.files." + driverId + ".type").equals(DataAccess.S3) && Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect"))) ||
-            Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
+        return (System.getProperty("dataverse.files." + driverId + ".type").equals(DataAccess.S3)
+                && Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")))
+                || Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
     }
-    
-    //Check that storageIdentifier is consistent with store's config
-    //False will prevent direct uploads
+
+    // Check that storageIdentifier is consistent with store's config
+    // False will prevent direct uploads
     static boolean isValidIdentifier(String driverId, String storageId) {
         return false;
     }
-    
-    //Utility to verify the standard UUID pattern for stored files.
+
+    // Utility to verify the standard UUID pattern for stored files.
     protected static boolean usesStandardNamePattern(String identifier) {
 
         Pattern r = Pattern.compile("^[a-f,0-9]{11}-[a-f,0-9]{12}$");
@@ -626,4 +628,15 @@ protected static boolean usesStandardNamePattern(String identifier) {
 
     public abstract List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOException;
 
+    /**
+     * A storage-type-specific mechanism for retrieving the size of a file. Intended
+     * primarily as a way to get the size before it has been recorded in the
+     * database, e.g. during direct/out-of-band transfers but could be useful to
+     * check the db values.
+     * 
+     * @return file size in bytes
+     * @throws IOException 
+     */
+    public abstract long retrieveSizeFromMedia() throws IOException;
+
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
index 6c84009de3e..0d1dab581fe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
@@ -954,4 +954,9 @@ public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOE
         }
         return toDelete;
     }
+
+    @Override
+    public long retrieveSizeFromMedia() throws IOException {
+        throw new UnsupportedDataAccessOperationException("InputStreamIO: this method is not supported in this DataAccess driver."); 
+    }
 }

From 68ab3f3cb6399d4c73bff0bcc84d9687ab369351 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 09:18:58 -0400
Subject: [PATCH 596/922] typos, change hash notice

---
 .../iq/dataverse/globus/GlobusServiceBean.java      | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 49572519696..8aa9915db58 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -4,8 +4,6 @@
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.GsonBuilder;
-import com.nimbusds.oauth2.sdk.pkce.CodeVerifier;
-
 import edu.harvard.iq.dataverse.*;
 
 import jakarta.ejb.Asynchronous;
@@ -21,7 +19,6 @@
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonPatch;
-import jakarta.json.JsonValue;
 import jakarta.servlet.http.HttpServletRequest;
 
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
@@ -662,6 +659,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
         
         ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
+        
         if(ruleId!=null) {
             Long datasetId = rulesCache.getIfPresent(ruleId);
             if(datasetId!=null) {
@@ -1095,7 +1093,7 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger)
         String fullPath = id.split("IDsplit")[1];
         String fileName = id.split("IDsplit")[2];
 
-        // ToDo: what if the file doesnot exists in s3
+        // ToDo: what if the file does not exist in s3
         // ToDo: what if checksum calculation failed
 
         do {
@@ -1107,8 +1105,8 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger)
             } catch (IOException ioex) {
                 count = 3;
                 logger.info(ioex.getMessage());
-                globusLogger.info("DataFile (fullPAth " + fullPath
-                        + ") does not appear to be accessible withing Dataverse: ");
+                globusLogger.info("DataFile (fullPath " + fullPath
+                        + ") does not appear to be accessible within Dataverse: ");
             } catch (Exception ex) {
                 count = count + 1;
                 ex.printStackTrace();
@@ -1119,7 +1117,7 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger)
         } while (count < 3);
 
         if (checksumVal.length() == 0) {
-            checksumVal = "NULL";
+            checksumVal = "Not available in Dataverse";
         }
 
         String mimeType = calculatemime(fileName);
@@ -1384,4 +1382,5 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
     private static boolean isDataverseManaged(String driverId) {
         return Boolean.getBoolean("dataverse.files." + driverId + ".managed");
     }
+    
 }

From d57b9f048490bcc2a38d8c2fc422e3797bad2fbc Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 09:19:24 -0400
Subject: [PATCH 597/922] add getLocationFromStorage, add tests

---
 .../iq/dataverse/dataaccess/DataAccess.java   | 34 +++++++++++++++----
 .../dataverse/dataaccess/DataAccessTest.java  | 20 +++++++++++
 2 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
index 8387f8110cf..a3345cb7a8c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
@@ -153,12 +153,34 @@ public static String[] getDriverIdAndStorageLocation(String storageLocation) {
     }
     
     public static String getStorageIdFromLocation(String location) {
-    	if(location.contains(SEPARATOR)) {
-    		//It's a full location with a driverId, so strip and reapply the driver id
-    		//NOte that this will strip the bucketname out (which s3 uses) but the S3IOStorage class knows to look at re-insert it
-    		return location.substring(0,location.indexOf(SEPARATOR) +3) + location.substring(location.lastIndexOf('/')+1); 
-    	}
-    	return location.substring(location.lastIndexOf('/')+1);
+        if (location.contains(SEPARATOR)) {
+            // It's a full location with a driverId, so strip and reapply the driver id
+            // NOte that this will strip the bucketname out (which s3 uses) but the
+            // S3IOStorage class knows to look at re-insert it
+            return location.substring(0, location.indexOf(SEPARATOR) + 3)
+                    + location.substring(location.lastIndexOf('/') + 1);
+        }
+        return location.substring(location.lastIndexOf('/') + 1);
+    }
+    
+    /** Changes storageidentifiers of the form
+     * s3://bucketname/18b39722140-50eb7d3c5ece or file://18b39722140-50eb7d3c5ece to s3://10.5072/FK2/ABCDEF/18b39722140-50eb7d3c5ece
+     * and
+     * 18b39722140-50eb7d3c5ece to 10.5072/FK2/ABCDEF/18b39722140-50eb7d3c5ece
+     * @param id
+     * @param dataset
+     * @return
+     */
+    public static String getLocationFromStorageId(String id, Dataset dataset) {
+        String path= dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/";
+        if (id.contains(SEPARATOR)) {
+            // It's a full location with a driverId, so strip and reapply the driver id
+            // NOte that this will strip the bucketname out (which s3 uses) but the
+            // S3IOStorage class knows to look at re-insert it
+            return id.substring(0, id.indexOf(SEPARATOR) + 3) + path
+                    + id.substring(id.lastIndexOf('/') + 1);
+        }
+        return path + id.substring(id.lastIndexOf('/') + 1);
     }
     
     public static String getDriverType(String driverId) {
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java
index 1ff914adff9..f7ce061fb24 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java
@@ -59,4 +59,24 @@ void testCreateNewStorageIO_createsFileAccessIObyDefault() throws IOException {
     StorageIO<Dataset> storageIo = DataAccess.createNewStorageIO(dataset, "valid-tag");
     assertTrue(storageIo.getClass().equals(FileAccessIO.class));
   }
+  
+  @Test
+  void testGetLocationFromStorageId() {
+      Dataset d = new Dataset();
+      d.setAuthority("10.5072");
+      d.setIdentifier("FK2/ABCDEF");
+      assertEquals("s3://10.5072/FK2/ABCDEF/18b39722140-50eb7d3c5ece",
+              DataAccess.getLocationFromStorageId("s3://18b39722140-50eb7d3c5ece", d));
+      assertEquals("10.5072/FK2/ABCDEF/18b39722140-50eb7d3c5ece",
+              DataAccess.getLocationFromStorageId("18b39722140-50eb7d3c5ece", d));
+
+  }
+  
+  @Test
+  void testGetStorageIdFromLocation() {
+      assertEquals("file://18b39722140-50eb7d3c5ece",
+              DataAccess.getStorageIdFromLocation("file://10.5072/FK2/ABCDEF/18b39722140-50eb7d3c5ece"));
+      assertEquals("s3://18b39722140-50eb7d3c5ece",
+              DataAccess.getStorageIdFromLocation("s3://bucketname:10.5072/FK2/ABCDEF/18b39722140-50eb7d3c5ece"));
+  }
 }

From 34286830d1cfa4849a82909eaff20528980fd717 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 09:19:53 -0400
Subject: [PATCH 598/922] get size for direct uploads

---
 .../impl/CreateNewDataFilesCommand.java       | 24 +++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index ac701da1be9..a8be1bd5116 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -3,18 +3,20 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
 import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
 import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
-//import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
 import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -83,7 +85,7 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private final String newStorageIdentifier; 
     private final String newCheckSum; 
     private DataFile.ChecksumType newCheckSumType;
-    private final Long newFileSize;
+    private Long newFileSize;
 
     public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum) {
         this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, null);
@@ -639,6 +641,24 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 }
                 logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType);
             }
+            if (newFileSize == null) {
+                // For direct/out-of-band upload, get the size from the underlying service
+                StorageIO<DvObject> sio;
+                try {
+                    sio = DataAccess.getDirectStorageIO(DataAccess.getLocationFromStorageId(newStorageIdentifier, version.getDataset()));
+
+                    // get file size
+                    // Note - some stores (e.g. AWS S3) only offer eventual consistency and a call
+                    // to get the size immediately after uploading may fail. As of the addition of
+                    // PR#9409 adding storage quotas, we are now requiring size to be available
+                    // earlier. If this is seen, adding
+                    // a delay/retry may help
+                    newFileSize = sio.retrieveSizeFromMedia();
+                } catch (IOException e) {
+                    // If we don't get a file size, a CommandExecutionException will be thrown later in the code
+                    e.printStackTrace();
+                }
+            }
         }
         // Finally, if none of the special cases above were applicable (or 
         // if we were unable to unpack an uploaded file, etc.), we'll just 

From 2adfa8af01124c31ada3f1801dd5f3dac0fd704e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 09:20:31 -0400
Subject: [PATCH 599/922] refactor, add delete method, etc.

---
 .../dataaccess/GlobusOverlayAccessIO.java     | 157 ++++++++++++------
 1 file changed, 110 insertions(+), 47 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 965dc3c0947..011bb74f720 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -17,11 +17,14 @@
 
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 
+import jakarta.json.Json;
 import jakarta.json.JsonObject;
+import jakarta.json.JsonObjectBuilder;
 
 /**
  * @author qqmyers
@@ -43,7 +46,6 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
-    String globusAccessToken = null;
     /*
      * If this is set to true, the store supports Globus transfer in and
      * Dataverse/the globus app manage file locations, access controls, deletion,
@@ -51,35 +53,64 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
      */
     private boolean dataverseManaged = false;
 
+    private String relativeDirectoryPath;
+    
+    private String endpointPath;
+    
+    private String filename;
+
+    private String endpoint;
+
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
-        if (dvObject instanceof DataFile) {
-            globusAccessToken = retrieveGlobusAccessToken();
-        }
         dataverseManaged = isDataverseManaged(this.driverId);
+    }
+
+    private void parsePath() {
+        int filenameStart = path.lastIndexOf("/") + 1;
+        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
+        int pathStart = endpointWithBasePath.indexOf("/");
+        logger.info("endpointWithBasePath: " + endpointWithBasePath);
+        endpointPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
+        logger.info("endpointPath: " + endpointPath);
+
+        if (dataverseManaged && (dvObject!=null)) {
+            
+            Dataset ds = null;
+            if (dvObject instanceof Dataset) {
+                ds = (Dataset) dvObject;
+            } else if (dvObject instanceof DataFile) {
+                ds = ((DataFile) dvObject).getOwner();
+            }
+            relativeDirectoryPath = "/" + ds.getAuthority() + "/" + ds.getIdentifier();
+        } else {
+            relativeDirectoryPath = "";
+        }
+        if (filenameStart > 0) {
+            relativeDirectoryPath = relativeDirectoryPath + path.substring(0, filenameStart);
+        }
+        logger.info("relativeDirectoryPath finally: " + relativeDirectoryPath);
+        filename = path.substring(filenameStart);
+        endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
 
-        logger.info("GAT3: " + globusAccessToken);
+        
     }
 
     public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
         this.driverId = driverId;
+        configureStores(null, driverId, storageLocation);
         this.dataverseManaged = isDataverseManaged(this.driverId);
         if (dataverseManaged) {
             String[] parts = DataAccess.getDriverIdAndStorageLocation(storageLocation);
             path = parts[1];
         } else {
             this.setIsLocalFile(false);
-            configureStores(null, driverId, storageLocation);
-
             path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
             validatePath(path);
             logger.fine("Relative path: " + path);
         }
-//ToDo - only when needed?
-        globusAccessToken = retrieveGlobusAccessToken();
-
     }
-
+    
     private String retrieveGlobusAccessToken() {
         // String globusToken = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
         String globusToken = System.getProperty("dataverse.files." + this.driverId + ".globus-token");
@@ -101,33 +132,16 @@ private void validatePath(String relPath) throws IOException {
 
     // Call the Globus API to get the file size
     @Override
-    long retrieveSize() {
+    public long retrieveSizeFromMedia() {
+        parsePath();
+        String globusAccessToken = retrieveGlobusAccessToken();
         logger.info("GAT2: " + globusAccessToken);
         // Construct Globus URL
         URI absoluteURI = null;
         try {
-            int filenameStart = path.lastIndexOf("/") + 1;
-            String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
-            int pathStart = endpointWithBasePath.indexOf("/");
-            logger.info("endpointWithBasePath: " + endpointWithBasePath);
-            String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
-            logger.info("directoryPath: " + directoryPath);
-
-            if (dataverseManaged && (dvObject!=null)) {
-                Dataset ds = ((DataFile) dvObject).getOwner();
-                directoryPath = directoryPath + "/" + ds.getAuthority() + "/" + ds.getIdentifier();
-                logger.info("directoryPath now: " + directoryPath);
-
-            }
-            if (filenameStart > 0) {
-                directoryPath = directoryPath + path.substring(0, filenameStart);
-            }
-            logger.info("directoryPath finally: " + directoryPath);
-            String filename = path.substring(filenameStart);
-            String endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
 
             absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint
-                    + "/ls?path=" + directoryPath + "&filter=name:" + filename);
+                    + "/ls?path=" + endpointPath + relativeDirectoryPath + "&filter=name:" + filename);
             HttpGet get = new HttpGet(absoluteURI);
 
             logger.info("Token is " + globusAccessToken);
@@ -166,25 +180,63 @@ public InputStream getInputStream() throws IOException {
     @Override
     public void delete() throws IOException {
 
-// Fix
-        // Delete is best-effort - we tell the remote server and it may or may not
-        // implement this call
+        parsePath();
+        // Delete is best-effort - we tell the endpoint to delete don't monitor whether
+        // it succeeds
         if (!isDirectAccess()) {
             throw new IOException("Direct Access IO must be used to permanently delete stored file objects");
         }
+        String globusAccessToken = retrieveGlobusAccessToken();
+        // Construct Globus URL
+        URI absoluteURI = null;
         try {
-            HttpDelete del = new HttpDelete(baseUrl + "/" + path);
-            CloseableHttpResponse response = getSharedHttpClient().execute(del, localContext);
-            try {
-                int code = response.getStatusLine().getStatusCode();
-                switch (code) {
+
+            absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/submission_id");
+            HttpGet get = new HttpGet(absoluteURI);
+
+            logger.info("Token is " + globusAccessToken);
+            get.addHeader("Authorization", "Bearer " + globusAccessToken);
+            CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
+            if (response.getStatusLine().getStatusCode() == 200) {
+                // Get reponse as string
+                String responseString = EntityUtils.toString(response.getEntity());
+                logger.info("Response from " + get.getURI().toString() + " is: " + responseString);
+                JsonObject responseJson = JsonUtil.getJsonObject(responseString);
+                String submissionId = responseJson.getString("value");
+                logger.info("submission_id for delete is: " + submissionId);
+                absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/delete");
+                HttpPost post = new HttpPost(absoluteURI);
+                JsonObjectBuilder taskJsonBuilder = Json.createObjectBuilder();
+                taskJsonBuilder.add("submission_id", submissionId).add("DATA_TYPE", "delete").add("endpoint", endpoint)
+                        .add("DATA", Json.createArrayBuilder().add(Json.createObjectBuilder().add("DATA_TYPE", "delete_item").add("path",
+                                endpointPath + relativeDirectoryPath + "/" + filename)));
+                post.setHeader("Content-Type", "application/json");
+                post.addHeader("Authorization", "Bearer " + globusAccessToken);
+                String taskJson= JsonUtil.prettyPrint(taskJsonBuilder.build());
+                logger.info("Sending: " + taskJson);
+                post.setEntity(new StringEntity(taskJson, "utf-8"));
+                CloseableHttpResponse postResponse = getSharedHttpClient().execute(post, localContext);
+                int statusCode=postResponse.getStatusLine().getStatusCode();
+                logger.info("Response :" + statusCode + ": " +postResponse.getStatusLine().getReasonPhrase());
+                switch (statusCode) {
+                case 202:
+                    // ~Success - delete task was accepted
+                    logger.info("Globus delete initiated: " + EntityUtils.toString(postResponse.getEntity()));
+                    break;
                 case 200:
-                    logger.fine("Sent DELETE for " + baseUrl + "/" + path);
+                    // Duplicate - delete task was already accepted
+                    logger.info("Duplicate Globus delete: " + EntityUtils.toString(postResponse.getEntity()));
+                    break;
                 default:
-                    logger.fine("Response from DELETE on " + del.getURI().toString() + " was " + code);
+                    logger.warning("Response from " + post.getURI().toString() + " was "
+                            + postResponse.getStatusLine().getStatusCode());
+                    logger.info(EntityUtils.toString(postResponse.getEntity()));
                 }
-            } finally {
-                EntityUtils.consume(response.getEntity());
+
+            } else {
+                logger.warning("Response from " + get.getURI().toString() + " was "
+                        + response.getStatusLine().getStatusCode());
+                logger.info(EntityUtils.toString(response.getEntity()));
             }
         } catch (Exception e) {
             logger.warning(e.getMessage());
@@ -250,6 +302,16 @@ static boolean isValidIdentifier(String driverId, String storageId) {
         return true;
     }
 
+    @Override
+    public String getStorageLocation() throws IOException {
+        parsePath();
+        if (dataverseManaged) {
+            return this.driverId + DataAccess.SEPARATOR + relativeDirectoryPath + "/" + filename;
+        } else {
+            return super.getStorageLocation();
+        }
+    }
+    
     public static void main(String[] args) {
         System.out.println("Running the main method");
         if (args.length > 0) {
@@ -272,7 +334,7 @@ public static void main(String[] args) {
         try {
             GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(
                     "globus://1234///hdc1/image001.mrc", "globus");
-            logger.info("Size is " + gsio.retrieveSize());
+            logger.info("Size is " + gsio.retrieveSizeFromMedia());
 
         } catch (IOException e) {
             // TODO Auto-generated catch block
@@ -286,7 +348,7 @@ public static void main(String[] args) {
             df.setOwner(ds);
             df.setStorageIdentifier("globus://1234///hdc1/image001.mrc");
             GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(df, null, "globus");
-            logger.info("Size2 is " + gsio.retrieveSize());
+            logger.info("Size2 is " + gsio.retrieveSizeFromMedia());
 
         } catch (IOException e) {
             // TODO Auto-generated catch block
@@ -294,4 +356,5 @@ public static void main(String[] args) {
         }
 
     }
+    
 }

From bdba5d8ef8a459314d5b8dccab30190461bbfdea Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 14:03:46 -0400
Subject: [PATCH 600/922] implement signedUrls for globus app, refactor

---
 .../edu/harvard/iq/dataverse/api/Admin.java   |  8 +-
 .../harvard/iq/dataverse/api/Datasets.java    | 75 ++++++++++++++++++-
 .../edu/harvard/iq/dataverse/api/Files.java   |  6 +-
 .../externaltools/ExternalToolHandler.java    | 71 +-----------------
 .../dataverse/globus/GlobusServiceBean.java   | 58 +++++++-------
 .../iq/dataverse/util/URLTokenUtil.java       | 65 ++++++++++++++++
 src/main/java/propertyFiles/Bundle.properties |  4 +-
 .../ExternalToolHandlerTest.java              | 11 +--
 .../ExternalToolServiceBeanTest.java          |  4 +-
 9 files changed, 192 insertions(+), 110 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index fd3b9a89e54..1870c7cb508 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -96,7 +96,6 @@
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteRoleCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteTemplateCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RegisterDvObjectCommand;
-import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.userdata.UserListMaker;
@@ -105,6 +104,7 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 
 import java.io.IOException;
@@ -2418,12 +2418,12 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
         }
         
         String baseUrl = urlInfo.getString("url");
-        int timeout = urlInfo.getInt(ExternalToolHandler.TIMEOUT, 10);
-        String method = urlInfo.getString(ExternalToolHandler.HTTP_METHOD, "GET");
+        int timeout = urlInfo.getInt(URLTokenUtil.TIMEOUT, 10);
+        String method = urlInfo.getString(URLTokenUtil.HTTP_METHOD, "GET");
         
         String signedUrl = UrlSignerUtil.signUrl(baseUrl, timeout, userId, method, key); 
         
-        return ok(Json.createObjectBuilder().add(ExternalToolHandler.SIGNED_URL, signedUrl));
+        return ok(Json.createObjectBuilder().add(URLTokenUtil.SIGNED_URL, signedUrl));
     }
  
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index d3ea1b80696..aad5a95bd8e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -90,6 +90,7 @@
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.MarkupChecker;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
 import edu.harvard.iq.dataverse.util.bagit.OREMap;
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
 import edu.harvard.iq.dataverse.util.json.JsonLDTerm;
@@ -3328,7 +3329,7 @@ public Response getTimestamps(@Context ContainerRequestContext crc, @PathParam("
 
     @POST
     @AuthRequired
-    @Path("{id}/addglobusFiles")
+    @Path("{id}/addGlobusFiles")
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
                                             @PathParam("id") String datasetId,
@@ -3411,6 +3412,74 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
 
     }
 
+    /**
+     * Retrieve the parameters and signed URLs required to perform a globus
+     * transfer. This api endpoint is expected to be called as a signed callback
+     * after the globus-dataverse app/other app is launched, but it will accept
+     * other forms of authentication.
+     * 
+     * @param crc
+     * @param datasetId
+     */
+    @GET
+    @AuthRequired
+    @Path("{id}/globusUploadParameters")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @QueryParam(value = "locale") String locale)
+    {
+        // -------------------------------------
+        // (1) Get the user from the ContainerRequestContext
+        // -------------------------------------
+        AuthenticatedUser authUser;
+        try {
+            authUser = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse e) {
+            return e.getResponse();
+        }
+        // -------------------------------------
+        // (2) Get the Dataset Id
+        // -------------------------------------
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(datasetId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        String storeId = dataset.getEffectiveStorageDriverId();
+        if(!DataAccess.getDriverType(storeId).equals(DataAccess.GLOBUS)) {
+            return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled"));
+        }
+        boolean managed = GlobusOverlayAccessIO.isDataverseManaged(storeId);
+        
+        JsonObjectBuilder queryParams = Json.createObjectBuilder();
+        queryParams.add("queryParameters",
+                Json.createArrayBuilder().add(Json.createObjectBuilder().add("datasetId", "{datasetId}"))
+                        .add(Json.createObjectBuilder().add("siteUrl", "{siteUrl}"))
+                        .add(Json.createObjectBuilder().add("datasetVersion", "{datasetVersion}"))
+                        .add(Json.createObjectBuilder().add("dvLocale", "{localeCode}"))
+                        .add(Json.createObjectBuilder().add("datasetPid", "{datasetPid}").add("managed", managed)));
+
+        JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
+                .add(URLTokenUtil.HTTP_METHOD, "POST")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusTransferPaths")
+                .add(URLTokenUtil.TIMEOUT, 300));
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addGlobusFiles")
+                .add(URLTokenUtil.HTTP_METHOD, "POST")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
+                .add(URLTokenUtil.TIMEOUT, 300));
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing")
+                .add(URLTokenUtil.HTTP_METHOD, "GET")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
+                .add(URLTokenUtil.TIMEOUT, 300));
+        
+
+        URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
+        return ok(tokenUtil.createPostBody(tokenUtil.getParams(queryParams.build()), allowedApiCalls.build()));
+    }
+    
     /** Requests permissions for a given globus user to upload to the dataset
      * 
      * @param crc
@@ -3915,8 +3984,8 @@ public Response getExternalToolDVParams(@Context ContainerRequestContext crc,
             }
             
 
-            ExternalToolHandler eth = new ExternalToolHandler(externalTool, target.getDataset(), apiToken, locale);
-            return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters()))));
+            URLTokenUtil eth = new ExternalToolHandler(externalTool, target.getDataset(), apiToken, locale);
+            return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters())), JsonUtil.getJsonArray(externalTool.getAllowedApiCalls())));
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index 82811162d52..4c2fa8f68ce 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -48,6 +48,8 @@
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
+
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
@@ -822,10 +824,10 @@ public Response getExternalToolFMParams(@Context ContainerRequestContext crc, @P
             return error(BAD_REQUEST, "FileMetadata not found.");
         }
 
-        ExternalToolHandler eth = null;
+        URLTokenUtil eth = null;
 
         eth = new ExternalToolHandler(externalTool, target.getDataFile(), apiToken, target, locale);
-        return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters()))));
+        return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters())), JsonUtil.getJsonArray(externalTool.getAllowedApiCalls())));
     }
     
     @GET
diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
index de4317464e6..36227c2f883 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
@@ -22,12 +22,8 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import jakarta.json.Json;
-import jakarta.json.JsonArray;
-import jakarta.json.JsonArrayBuilder;
 import jakarta.json.JsonNumber;
 import jakarta.json.JsonObject;
-import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonString;
 import jakarta.json.JsonValue;
 import jakarta.ws.rs.HttpMethod;
@@ -41,15 +37,10 @@
  */
 public class ExternalToolHandler extends URLTokenUtil {
 
-    private final ExternalTool externalTool;
+    public final ExternalTool externalTool;
 
     private String requestMethod;
-    
-    public static final String HTTP_METHOD="httpMethod";
-    public static final String TIMEOUT="timeOut";
-    public static final String SIGNED_URL="signedUrl";
-    public static final String NAME="name";
-    public static final String URL_TEMPLATE="urlTemplate";
+
     
 
     /**
@@ -134,10 +125,10 @@ public String handleRequest(boolean preview) {
 
         } else {
             // ToDo - if the allowedApiCalls() are defined, could/should we send them to
-            // tools using GET as well?
+            // tools using POST as well?
 
             if (requestMethod.equals(HttpMethod.POST)) {
-                String body = JsonUtil.prettyPrint(createPostBody(params).build());
+                String body = JsonUtil.prettyPrint(createPostBody(params, null).build());
                 try {
                     logger.info("POST Body: " + body);
                     return postFormData(body);
@@ -149,60 +140,6 @@ public String handleRequest(boolean preview) {
         return null;
     }
 
-    public JsonObject getParams(JsonObject toolParameters) {
-        //ToDo - why an array of object each with a single key/value pair instead of one object?
-        JsonArray queryParams = toolParameters.getJsonArray("queryParameters");
-
-        // ToDo return json and print later
-        JsonObjectBuilder paramsBuilder = Json.createObjectBuilder();
-        if (!(queryParams == null) && !queryParams.isEmpty()) {
-            queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> {
-                queryParam.keySet().forEach((key) -> {
-                    String value = queryParam.getString(key);
-                    JsonValue param = getParam(value);
-                    if (param != null) {
-                        paramsBuilder.add(key, param);
-                    }
-                });
-            });
-        }
-        return paramsBuilder.build();
-    }
-
-    public JsonObjectBuilder createPostBody(JsonObject params) {
-        JsonObjectBuilder bodyBuilder = Json.createObjectBuilder();
-        bodyBuilder.add("queryParameters", params);
-        String apiCallStr = externalTool.getAllowedApiCalls();
-        if (apiCallStr != null && !apiCallStr.isBlank()) {
-            JsonArray apiArray = JsonUtil.getJsonArray(externalTool.getAllowedApiCalls());
-            JsonArrayBuilder apisBuilder = Json.createArrayBuilder();
-            apiArray.getValuesAs(JsonObject.class).forEach(((apiObj) -> {
-                logger.fine(JsonUtil.prettyPrint(apiObj));
-                String name = apiObj.getJsonString(NAME).getString();
-                String httpmethod = apiObj.getJsonString(HTTP_METHOD).getString();
-                int timeout = apiObj.getInt(TIMEOUT);
-                String urlTemplate = apiObj.getJsonString(URL_TEMPLATE).getString();
-                logger.fine("URL Template: " + urlTemplate);
-                urlTemplate = SystemConfig.getDataverseSiteUrlStatic() + urlTemplate;
-                String apiPath = replaceTokensWithValues(urlTemplate);
-                logger.fine("URL WithTokens: " + apiPath);
-                String url = apiPath;
-                // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users)
-                ApiToken apiToken = getApiToken();
-                if (apiToken != null) {
-                    url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(),
-                            httpmethod, JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("")
-                                    + getApiToken().getTokenString());
-                }
-                logger.fine("Signed URL: " + url);
-                apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod)
-                        .add(SIGNED_URL, url).add(TIMEOUT, timeout));
-            }));
-            bodyBuilder.add("signedUrls", apisBuilder);
-        }
-        return bodyBuilder;
-    }
-
     private String postFormData(String allowedApis) throws IOException, InterruptedException {
         String url = null;
         HttpClient client = HttpClient.newHttpClient();
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 8aa9915db58..2c0edd070f3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -20,6 +20,7 @@
 import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonPatch;
 import jakarta.servlet.http.HttpServletRequest;
+import jakarta.ws.rs.HttpMethod;
 
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json;
 import static edu.harvard.iq.dataverse.util.json.JsonPrinter.toJsonArray;
@@ -45,6 +46,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import org.apache.commons.codec.binary.StringUtils;
+
 import com.google.gson.Gson;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
@@ -58,6 +61,7 @@
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.URLTokenUtil;
+import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 @Stateless
@@ -120,7 +124,6 @@ private String getRuleId(GlobusEndpoint endpoint, String principal, String permi
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access_list");
         MakeRequestResponse result = makeRequest(url, "Bearer",
                 endpoint.getClientToken(), "GET", null);
-        ArrayList<String> ids = new ArrayList<String>();
         if (result.status == 200) {
             AccessList al = parseJson(result.jsonResponse, AccessList.class, false);
 
@@ -282,7 +285,7 @@ private String getUniqueFilePath(GlobusEndpoint endpoint) {
     //Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
             .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
-            .removalListener((ruleId, datasetId, cause) -> {
+            .evictionListener((ruleId, datasetId, cause) -> {
                 //Delete rules that expire
                 Dataset dataset = datasetSvc.find(datasetId);
                 deletePermission((String) ruleId, dataset, null);
@@ -575,12 +578,23 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
         } catch (Exception e) {
             logger.warning("GlobusAppUrlForDataset: Failed to get storePrefix for " + driverId);
         }
-        //Use URLTokenUtil for params currently in common with external tools. 
+        // Use URLTokenUtil for params currently in common with external tools.
         URLTokenUtil tokenUtil = new URLTokenUtil(d, df, apiToken, localeCode);
         String appUrl;
         if (upload) {
             appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
-                    + "/upload?datasetPid={datasetPid}&siteUrl={siteUrl}&apiToken={apiToken}&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}";
+                    + "/upload?datasetPid={datasetPid}&siteUrl={siteUrl}&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}";
+            String callback = SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + d.getId()
+                    + "/globusUploadParameters?locale=" + localeCode;
+            if (apiToken != null) {
+                callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(),
+                        HttpMethod.GET,
+                        JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString());
+            } else {
+                // Shouldn't happen
+                logger.warning("unable to get api token for user: " + user.getIdentifier());
+            }
+            appUrl = appUrl + "&callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback));
         } else {
             if (df == null) {
                 appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
@@ -637,39 +651,27 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
             globusLogger = logger;
         }
 
-        globusLogger.info("Starting an globusUpload ");
+        Thread.sleep(5000);
 
-        
-        // ToDo - use DataAccess methods?
-        //String storageType = datasetIdentifier.substring(0, datasetIdentifier.indexOf("://") + 3);
-        //datasetIdentifier = datasetIdentifier.substring(datasetIdentifier.indexOf("://") + 3);
-        
         logger.fine("json: " + JsonUtil.prettyPrint(jsonData));
 
         String taskIdentifier = jsonData.getString("taskIdentifier");
 
-        String ruleId = null;
-
-        Thread.sleep(5000);
-        
         // globus task status check
         GlobusTask task = globusStatusCheck(taskIdentifier, globusLogger);
         String taskStatus = getTaskStatus(task);
 
+        globusLogger.info("Starting an globusUpload ");
+
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
-        
-        ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
-        
-        if(ruleId!=null) {
+        String ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
+        logger.info("Found rule: " + ruleId);
+        if (ruleId != null) {
             Long datasetId = rulesCache.getIfPresent(ruleId);
-            if(datasetId!=null) {
-             
-            //Will delete rule
-            rulesCache.invalidate(ruleId);
-            } else {
-                //The cache already expired this rule, in which case it's delay not long enough, or we have some other problem
-                logger.warning("Rule " + ruleId + " not found in rulesCache");
-                deletePermission(ruleId, dataset, globusLogger);
+            if (datasetId != null) {
+
+                // Will delete rule
+                rulesCache.invalidate(ruleId);
             }
         }
 
@@ -836,6 +838,10 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                 datasetSvc.removeDatasetLocks(dataset, DatasetLock.Reason.EditInProgress);
             }
         }
+        if (ruleId != null) {
+            deletePermission(ruleId, dataset, globusLogger);
+            globusLogger.info("Removed upload permission: " + ruleId);
+        }
     }
 
     public String addFilesAsync(String curlCommand, Logger globusLogger)
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java
index 4ae76a7b8db..216237105aa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java
@@ -6,6 +6,10 @@
 import java.util.regex.Pattern;
 
 import jakarta.json.Json;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonArrayBuilder;
+import jakarta.json.JsonObject;
+import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonValue;
 
 import edu.harvard.iq.dataverse.DataFile;
@@ -13,6 +17,8 @@
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 public class URLTokenUtil {
 
@@ -22,6 +28,13 @@ public class URLTokenUtil {
     protected final FileMetadata fileMetadata;
     protected ApiToken apiToken;
     protected String localeCode;
+    
+    
+    public static final String HTTP_METHOD="httpMethod";
+    public static final String TIMEOUT="timeOut";
+    public static final String SIGNED_URL="signedUrl";
+    public static final String NAME="name";
+    public static final String URL_TEMPLATE="urlTemplate";
 
     /**
      * File level
@@ -193,6 +206,58 @@ private String getTokenValue(String value) {
         throw new IllegalArgumentException("Cannot replace reserved word: " + value);
     }
     
+    public JsonObjectBuilder createPostBody(JsonObject params, JsonArray allowedApiCalls) {
+        JsonObjectBuilder bodyBuilder = Json.createObjectBuilder();
+        bodyBuilder.add("queryParameters", params);
+        if (allowedApiCalls != null && !allowedApiCalls.isEmpty()) {
+            JsonArrayBuilder apisBuilder = Json.createArrayBuilder();
+            allowedApiCalls.getValuesAs(JsonObject.class).forEach(((apiObj) -> {
+                logger.fine(JsonUtil.prettyPrint(apiObj));
+                String name = apiObj.getJsonString(NAME).getString();
+                String httpmethod = apiObj.getJsonString(HTTP_METHOD).getString();
+                int timeout = apiObj.getInt(TIMEOUT);
+                String urlTemplate = apiObj.getJsonString(URL_TEMPLATE).getString();
+                logger.fine("URL Template: " + urlTemplate);
+                urlTemplate = SystemConfig.getDataverseSiteUrlStatic() + urlTemplate;
+                String apiPath = replaceTokensWithValues(urlTemplate);
+                logger.fine("URL WithTokens: " + apiPath);
+                String url = apiPath;
+                // Sign if apiToken exists, otherwise send unsigned URL (i.e. for guest users)
+                ApiToken apiToken = getApiToken();
+                if (apiToken != null) {
+                    url = UrlSignerUtil.signUrl(apiPath, timeout, apiToken.getAuthenticatedUser().getUserIdentifier(),
+                            httpmethod, JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("")
+                                    + getApiToken().getTokenString());
+                }
+                logger.fine("Signed URL: " + url);
+                apisBuilder.add(Json.createObjectBuilder().add(NAME, name).add(HTTP_METHOD, httpmethod)
+                        .add(SIGNED_URL, url).add(TIMEOUT, timeout));
+            }));
+            bodyBuilder.add("signedUrls", apisBuilder);
+        }
+        return bodyBuilder;
+    }
+
+    public JsonObject getParams(JsonObject toolParameters) {
+        //ToDo - why an array of object each with a single key/value pair instead of one object?
+        JsonArray queryParams = toolParameters.getJsonArray("queryParameters");
+    
+        // ToDo return json and print later
+        JsonObjectBuilder paramsBuilder = Json.createObjectBuilder();
+        if (!(queryParams == null) && !queryParams.isEmpty()) {
+            queryParams.getValuesAs(JsonObject.class).forEach((queryParam) -> {
+                queryParam.keySet().forEach((key) -> {
+                    String value = queryParam.getString(key);
+                    JsonValue param = getParam(value);
+                    if (param != null) {
+                        paramsBuilder.add(key, param);
+                    }
+                });
+            });
+        }
+        return paramsBuilder.build();
+    }
+
     public static String getScriptForUrl(String url) {
         String msg = BundleUtil.getStringFromBundle("externaltools.enable.browser.popups");
         String script = "const newWin = window.open('" + url + "', target='_blank'); if (!newWin || newWin.closed || typeof newWin.closed == \"undefined\") {alert(\"" + msg + "\");}";
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 3497b23eb94..88f819b417b 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2645,8 +2645,8 @@ datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymi
 datasets.api.creationdate=Date Created
 datasets.api.modificationdate=Last Modified Date
 datasets.api.curationstatus=Curation Status
-datasets.api.globusdownloaddisabled=File transfer from Dataverse via Globus is not available for this installation of Dataverse.
-datasets.api.globusuploaddisabled=File transfer to Dataverse via Globus is not available for this installation of Dataverse.
+datasets.api.globusdownloaddisabled=File transfer from Dataverse via Globus is not available for this dataset.
+datasets.api.globusuploaddisabled=File transfer to Dataverse via Globus is not available for this dataset.
 
 
 
diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
index 21bb6633204..6f0132e2bc9 100644
--- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandlerTest.java
@@ -10,6 +10,7 @@
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.testing.JvmSetting;
 import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
@@ -53,7 +54,7 @@ public void testGetToolUrlWithOptionalQueryParameters() {
         Exception expectedException1 = null;
         String nullLocaleCode = null;
         try {
-            ExternalToolHandler externalToolHandler1 = new ExternalToolHandler(externalTool, nullDataFile, nullApiToken, nullFileMetadata, nullLocaleCode);
+            URLTokenUtil externalToolHandler1 = new ExternalToolHandler(externalTool, nullDataFile, nullApiToken, nullFileMetadata, nullLocaleCode);
         } catch (Exception ex) {
             expectedException1 = ex;
         }
@@ -71,7 +72,7 @@ public void testGetToolUrlWithOptionalQueryParameters() {
         DataFile dataFile = new DataFile();
         dataFile.setId(42l);
         try {
-            ExternalToolHandler externalToolHandler1 = new ExternalToolHandler(externalTool, dataFile, nullApiToken, nullFileMetadata, nullLocaleCode);
+            URLTokenUtil externalToolHandler1 = new ExternalToolHandler(externalTool, dataFile, nullApiToken, nullFileMetadata, nullLocaleCode);
         } catch (Exception ex) {
             expectedException1 = ex;
         }
@@ -92,7 +93,7 @@ public void testGetToolUrlWithOptionalQueryParameters() {
                 .build().toString());
         Exception expectedException2 = null;
         try {
-            ExternalToolHandler externalToolHandler2 = new ExternalToolHandler(externalTool, nullDataFile, nullApiToken, nullFileMetadata, nullLocaleCode);
+            URLTokenUtil externalToolHandler2 = new ExternalToolHandler(externalTool, nullDataFile, nullApiToken, nullFileMetadata, nullLocaleCode);
         } catch (Exception ex) {
             expectedException2 = ex;
         }
@@ -225,10 +226,10 @@ public void testGetToolUrlWithAllowedApiCalls() {
         assertTrue(et != null);
         System.out.println("allowedApiCalls et created");
         System.out.println(et.getAllowedApiCalls());
-        ExternalToolHandler externalToolHandler = new ExternalToolHandler(et, ds, at, null);
+        URLTokenUtil externalToolHandler = new ExternalToolHandler(et, ds, at, null);
         System.out.println("allowedApiCalls eth created");
         JsonObject jo = externalToolHandler
-                .createPostBody(externalToolHandler.getParams(JsonUtil.getJsonObject(et.getToolParameters()))).build();
+                .createPostBody(externalToolHandler.getParams(JsonUtil.getJsonObject(et.getToolParameters())), JsonUtil.getJsonArray(et.getAllowedApiCalls())).build();
         assertEquals(1, jo.getJsonObject("queryParameters").getInt("datasetId"));
         String signedUrl = jo.getJsonArray("signedUrls").getJsonObject(0).getString("signedUrl");
         // The date and token will change each time but check for the constant parts of
diff --git a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java
index 9337949f605..4f5af8b97b0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/externaltools/ExternalToolServiceBeanTest.java
@@ -9,6 +9,8 @@
 import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
+
 import java.util.ArrayList;
 import java.util.List;
 import jakarta.json.Json;
@@ -49,7 +51,7 @@ public void testfindAll() {
         externalToolTypes.add(externalToolType);
         ExternalTool.Scope scope = ExternalTool.Scope.FILE;
         ExternalTool externalTool = new ExternalTool("displayName", "toolName", "description", externalToolTypes, scope, "http://foo.com", "{}", DataFileServiceBean.MIME_TYPE_TSV_ALT);
-        ExternalToolHandler externalToolHandler4 = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, null);
+        URLTokenUtil externalToolHandler4 = new ExternalToolHandler(externalTool, dataFile, apiToken, fmd, null);
         List<ExternalTool> externalTools = new ArrayList<>();
         externalTools.add(externalTool);
         List<ExternalTool> availableExternalTools = externalToolService.findExternalToolsByFile(externalTools, dataFile);

From 520d5d6bdbea31ad56d465b7a2b4a1fdfaf40bb6 Mon Sep 17 00:00:00 2001
From: Saikiran Patil <patilrsaikiran@gmail.com>
Date: Fri, 20 Oct 2023 00:24:13 +0530
Subject: [PATCH 601/922] correcting the Invalid JSON error

---
 doc/sphinx-guides/source/api/native-api.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 1e0804ce7d8..73a10f2b409 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3108,7 +3108,7 @@ A curl example using an ``ID``
   export ID=24
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false},"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false,"dataFileTags":["Survey"]}' \
     "$SERVER_URL/api/files/$ID/metadata"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3116,7 +3116,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false},"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false,"dataFileTags":["Survey"]}' \
     "http://demo.dataverse.org/api/files/24/metadata"
 
 A curl example using a ``PERSISTENT_ID``
@@ -3128,7 +3128,7 @@ A curl example using a ``PERSISTENT_ID``
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false},"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false,"dataFileTags":["Survey"]}' \
     "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3136,7 +3136,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}', "dataFileTags":["Survey"]} \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false, "dataFileTags":["Survey"]} \
     "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
 
 Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.

From df7a4cfb9597b3c224a8d753b3a84d57d643af34 Mon Sep 17 00:00:00 2001
From: Saikiran Patil <patilrsaikiran@gmail.com>
Date: Fri, 20 Oct 2023 00:36:20 +0530
Subject: [PATCH 602/922] minor change

---
 doc/sphinx-guides/source/api/native-api.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 73a10f2b409..fcd2594ac6a 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3108,7 +3108,7 @@ A curl example using an ``ID``
   export ID=24
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false,"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false,"dataFileTags":["Survey"]}' \
     "$SERVER_URL/api/files/$ID/metadata"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3116,7 +3116,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false,"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false,"dataFileTags":["Survey"]}' \
     "http://demo.dataverse.org/api/files/24/metadata"
 
 A curl example using a ``PERSISTENT_ID``
@@ -3128,7 +3128,7 @@ A curl example using a ``PERSISTENT_ID``
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false,"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false,"dataFileTags":["Survey"]}' \
     "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3136,7 +3136,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false, "dataFileTags":["Survey"]} \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false, "dataFileTags":["Survey"]} \
     "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
 
 Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.

From 8c9828a3fa0c2d534ebcfc57f4cc17427e75f0b8 Mon Sep 17 00:00:00 2001
From: Saikiran Patil <patilrsaikiran@gmail.com>
Date: Fri, 20 Oct 2023 00:39:49 +0530
Subject: [PATCH 603/922] another small change

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index fcd2594ac6a..ffb15b41fd1 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3136,7 +3136,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false, "dataFileTags":["Survey"]} \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false, "dataFileTags":["Survey"]}' \
     "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
 
 Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.

From f056d6c051bf784ca4808e8757efa9afcaf7778c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 19 Oct 2023 15:10:14 -0400
Subject: [PATCH 604/922] minor incremental changes (#9635)

---
 .../search/SearchIncludeFragment.java         | 30 +++++++++++++++----
 .../dataverse/search/SearchServiceBean.java   |  6 ++--
 .../dataverse/search/SolrQueryResponse.java   | 10 ++++++-
 src/main/webapp/search-include-fragment.xhtml | 24 +++++++++++++--
 4 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 47a5621c3d6..14274a09399 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -131,7 +131,8 @@ public class SearchIncludeFragment implements java.io.Serializable {
     Map<String, String> datasetfieldFriendlyNamesBySolrField = new HashMap<>();
     Map<String, String> staticSolrFieldFriendlyNamesBySolrField = new HashMap<>();
     private boolean solrIsDown = false;
-    private boolean solrIsOverloaded = false; 
+    private boolean solrIsTemporarilyUnavailable = false; 
+    private boolean solrFacetsDisabled = false;
     private Map<String, Integer> numberOfFacets = new HashMap<>();
 //    private boolean showUnpublished;
     List<String> filterQueriesDebug = new ArrayList<>();
@@ -361,6 +362,14 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             if (solrQueryResponse.hasError()){
                 logger.info(solrQueryResponse.getError());
                 setSolrErrorEncountered(true);
+            } 
+            // Solr "temporarily unavailable" is the condition triggered by 
+            // receiving a 503 from the search engine, that is in turn a result
+            // of one of the Solr "circuit breakers" being triggered by excessive
+            // load. We treat this condition as distinct from "Solr is down", 
+            // on the assumption that it is transitive. 
+            if (solrQueryResponse.isSolrTemporarilyUnavailable()) {
+                setSolrTemporarilyUnavailable(true);
             }
             // This 2nd search() is for populating the "type" ("dataverse", "dataset", "file") facets: -- L.A. 
             // (why exactly do we need it, again?)
@@ -386,7 +395,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 }
             }
             
-            if (selectedTypesList.size() < 3) {
+            if (selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable()) {
                 // If some types are NOT currently selected, we will need to 
                 // run another query to obtain the numbers of the unselected types:
                 
@@ -1079,14 +1088,23 @@ public void setSolrIsDown(boolean solrIsDown) {
         this.solrIsDown = solrIsDown;
     }
     
-    public boolean isSolrOverloaded() {
-        return solrIsOverloaded;
+    public boolean isSolrTemporarilyUnavailable() {
+        return solrIsTemporarilyUnavailable;
     }
     
-    public void setSolrIsOverloaded(boolean solrIsOverloaded) {
-        this.solrIsOverloaded = solrIsOverloaded; 
+    public void setSolrTemporarilyUnavailable(boolean solrIsTemporarilyUnavailable) {
+        this.solrIsTemporarilyUnavailable = solrIsTemporarilyUnavailable; 
     }
 
+    public boolean isFacetsDisabled() {
+        return solrFacetsDisabled;
+    }
+    
+    public void setFacetsDisabled(boolean solrFacetsDisabled) {
+        this.solrFacetsDisabled = solrFacetsDisabled; 
+    }
+    
+    
     public boolean isRootDv() {
         return rootDv;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 1b92c2a4a46..6e410488794 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -382,7 +382,6 @@ public SolrQueryResponse search(
         // Make the solr query
         // -----------------------------------
         QueryResponse queryResponse = null;
-        boolean solrTemporarilyUnavailable = false; 
         
         try {
             queryResponse = solrClientService.getSolrClient().query(solrQuery);
@@ -397,6 +396,8 @@ public SolrQueryResponse search(
                 logger.info("return code: "+queryResponse.getStatus());
             }
             
+            SolrQueryResponse exceptionSolrQueryResponse = new SolrQueryResponse(solrQuery);
+
             // We probably shouldn't be assuming that this is necessarily a 
             // "search syntax error", as the code below implies - could be 
             // something else too - ? 
@@ -407,9 +408,9 @@ public SolrQueryResponse search(
             // a transient condition): 
             
             if (ex.code() == 503) {
-                solrTemporarilyUnavailable = true;
                 // actual logic for communicating this state back to the local 
                 // client code TBD (@todo)
+                exceptionSolrQueryResponse.setSolrTemporarilyUnavailable(true);
             }
             
             String error = "Search Syntax Error: ";
@@ -421,7 +422,6 @@ public SolrQueryResponse search(
                 error += messageFromSolr;
             }
             logger.info(error);
-            SolrQueryResponse exceptionSolrQueryResponse = new SolrQueryResponse(solrQuery);
             exceptionSolrQueryResponse.setError(error);
 
             // we can't show anything because of the search syntax error
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrQueryResponse.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrQueryResponse.java
index 893099ff08d..27e79cb1fc2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SolrQueryResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrQueryResponse.java
@@ -26,6 +26,7 @@ public class SolrQueryResponse {
     private String error;
     private Map<String, Long> dvObjectCounts = new HashMap<>();
     private Map<String, Long> publicationStatusCounts = new HashMap<>();
+    private boolean solrTemporarilyUnavailable = false;
 
     public static String DATAVERSES_COUNT_KEY = "dataverses_count";
     public static String DATASETS_COUNT_KEY = "datasets_count";
@@ -91,7 +92,14 @@ public JsonObjectBuilder getPublicationStatusCountsAsJSON(){
         }
         return this.getMapCountsAsJSON(publicationStatusCounts);
     }
-       
+    
+    public boolean isSolrTemporarilyUnavailable() {
+        return solrTemporarilyUnavailable;
+    }
+    
+    public void setSolrTemporarilyUnavailable(boolean solrTemporarilyUnavailable) {
+        this.solrTemporarilyUnavailable = solrTemporarilyUnavailable;
+    }
     
     public JsonObjectBuilder getDvObjectCountsAsJSON(){
         
diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml
index 718df813348..8397a14136e 100644
--- a/src/main/webapp/search-include-fragment.xhtml
+++ b/src/main/webapp/search-include-fragment.xhtml
@@ -88,12 +88,24 @@
             <pre><h:outputText value="#{SearchIncludeFragment.searchException.cause.message}"/></pre>
         </div>
     </div>
+    
+    
+    <!-- ... And the message block for when SOLR IS DOWN *TEMPORARILY* -->
+    <div class="row" jsf:rendered="#{SearchIncludeFragment.solrTemporarilyUnavailable}">
+        <div class="col-sm-12">
+            <div class="bg-danger padding-12 margin-bottom">
+                <!-- @todo: move this text into the bundle -->
+                <h:outputText value="Search Engine service (Solr) is temporarily unavailable because of load issues. Please try again later. " styleClass="highlightBold"/>
+            </div>
+            <pre><h:outputText value="Note that all the datasets that are part of this collection are accessible via direct links and registered DOIs"/></pre>
+        </div>
+    </div>
 
     <!-- Search Results / SOLR IS UP -->
-    <div id="search-row-panel" class="row row-offcanvas row-offcanvas-left" jsf:rendered="#{!SearchIncludeFragment.solrIsDown}">
+    <div id="search-row-panel" class="row row-offcanvas row-offcanvas-left" jsf:rendered="#{!(SearchIncludeFragment.solrIsDown or SearchIncludeFragment.solrTemporarilyUnavailable)}">
         <div id="dv-sidebar" class="col-sm-4 col-md-3 sidebar-offcanvas">
             <!--TYPE FACET (DATAVERSES, DATASETS, FILES)-->
-            <h:form id="facetType">
+            <h:form id="facetType" rendered="#{!SearchIncludeFragment.facetsDisabled}">
                 <!--DATAVERSE TOGGLE-->
                 <div class="clearfix">
                     <h:outputLink value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('dataverses') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('dataverses') == null}">
@@ -193,7 +205,7 @@
             </h:form>
 
             <!--NON-TYPE FACETS-->
-            <h:form id="facetCategoryForm" rendered="#{DataversePage.searchResultsCount > 0}">
+            <h:form id="facetCategoryForm" rendered="#{(DataversePage.searchResultsCount > 0) and !SearchIncludeFragment.facetsDisabled}">
                 <p:dataList id="facetCategoryList" value="#{DataversePage.facetCategoryList}" var="facetCategory">
                     <h:outputText value="#{facetCategory.friendlyName}" styleClass="facetCategoryName"/>
                     <p:dataList value="#{facetCategory.facetLabel}" var="facetLabel" rows="#{DataversePage.getNumberOfFacets(facetCategory.name,5)}">
@@ -251,6 +263,12 @@
                     </ui:fragment>
                 </p:dataList>
             </h:form>
+            <!-- "facets unavailable" message: -->
+            <h:form id="facetsUnavailable" rendered="#{SearchIncludeFragment.facetsDisabled}">
+                <div class="clearfix">
+                    <h:outputText value="Facets temporarily unavailable" styleClass="facetTypeLink"/>
+                </div>
+            </h:form>
         </div>
         <div id="dv-main" class="col-sm-8 col-md-9">
             <!--DEBUG BEGIN-->

From 8f53aff901e678f3a8746b8a273c0e4ec811df5b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 19 Oct 2023 15:17:54 -0400
Subject: [PATCH 605/922] send endpoint, refactor getting config params

---
 .../harvard/iq/dataverse/api/Datasets.java    |  6 +--
 .../iq/dataverse/dataaccess/DataAccess.java   |  6 +--
 .../iq/dataverse/dataaccess/FileAccessIO.java |  3 +-
 .../dataaccess/GlobusOverlayAccessIO.java     | 25 ++++++++---
 .../dataaccess/RemoteOverlayAccessIO.java     | 37 +++++++++-------
 .../iq/dataverse/dataaccess/S3AccessIO.java   | 34 +++++++++------
 .../iq/dataverse/dataaccess/StorageIO.java    | 42 ++++++++++++++++---
 .../dataverse/dataaccess/SwiftAccessIO.java   | 35 ++++++++++------
 8 files changed, 131 insertions(+), 57 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index aad5a95bd8e..73c00c1d21b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3452,14 +3452,14 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
             return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled"));
         }
         boolean managed = GlobusOverlayAccessIO.isDataverseManaged(storeId);
-        
+        String endpoint = GlobusOverlayAccessIO.getEndpointId(storeId);
         JsonObjectBuilder queryParams = Json.createObjectBuilder();
         queryParams.add("queryParameters",
                 Json.createArrayBuilder().add(Json.createObjectBuilder().add("datasetId", "{datasetId}"))
                         .add(Json.createObjectBuilder().add("siteUrl", "{siteUrl}"))
                         .add(Json.createObjectBuilder().add("datasetVersion", "{datasetVersion}"))
-                        .add(Json.createObjectBuilder().add("dvLocale", "{localeCode}"))
-                        .add(Json.createObjectBuilder().add("datasetPid", "{datasetPid}").add("managed", managed)));
+                        .add(Json.createObjectBuilder().add("dvLocale", "{localeCode}")).add(Json.createObjectBuilder()
+                                .add("datasetPid", "{datasetPid}").add("managed", managed).add("endpoint", endpoint)));
 
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
index a3345cb7a8c..4eaa5f9269c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
@@ -48,7 +48,7 @@ public DataAccess() {
     public static final String S3 = "s3";
     static final String SWIFT = "swift";
     static final String REMOTE = "remote";
-    static final String GLOBUS = "globus";
+    public static final String GLOBUS = "globus";
     static final String TMP = "tmp";
     public static final String SEPARATOR = "://";
     //Default to "file" is for tests only
@@ -187,7 +187,7 @@ public static String getDriverType(String driverId) {
     	if(driverId.isEmpty() || driverId.equals("tmp")) {
     		return "tmp";
     	}
-    	return System.getProperty("dataverse.files." + driverId + ".type", "Undefined");
+    	return StorageIO.getConfigParamForDriver(driverId, StorageIO.TYPE, "Undefined");
     }
     
     //This 
@@ -195,7 +195,7 @@ public static String getDriverPrefix(String driverId) throws IOException {
         if(driverId.isEmpty() || driverId.equals("tmp")) {
             return "tmp" + SEPARATOR;
         }
-        String storageType = System.getProperty("dataverse.files." + driverId + ".type", "Undefined");
+        String storageType = StorageIO.getConfigParamForDriver(driverId, StorageIO.TYPE, "Undefined");
         switch(storageType) {
         case FILE:
             return FileAccessIO.getDriverPrefix(driverId);
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
index 3e6c802c526..f2a1312a150 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java
@@ -53,6 +53,7 @@
 public class FileAccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.FileAccessIO");
+    public static final String DIRECTORY = "directory";
 
 
     public FileAccessIO() {
@@ -578,7 +579,7 @@ private String getDatasetDirectory() throws IOException {
     
     
     protected String getFilesRootDirectory() {
-        String filesRootDirectory = System.getProperty("dataverse.files." + this.driverId + ".directory", "/tmp/files");
+        String filesRootDirectory = getConfigParam(DIRECTORY, "/tmp/files");
         return filesRootDirectory;
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 011bb74f720..502340feb7c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -46,6 +46,9 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
+    static final String GLOBUS_TOKEN = "globus-token";
+    static final String MANAGED = "managed";
+
     /*
      * If this is set to true, the store supports Globus transfer in and
      * Dataverse/the globus app manage file locations, access controls, deletion,
@@ -68,7 +71,7 @@ public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId)
 
     private void parsePath() {
         int filenameStart = path.lastIndexOf("/") + 1;
-        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
+        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf(DataAccess.SEPARATOR) + 3);
         int pathStart = endpointWithBasePath.indexOf("/");
         logger.info("endpointWithBasePath: " + endpointWithBasePath);
         endpointPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
@@ -112,13 +115,15 @@ public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOE
     }
     
     private String retrieveGlobusAccessToken() {
-        // String globusToken = JvmSettings.GLOBUS_TOKEN.lookup(driverId);
-        String globusToken = System.getProperty("dataverse.files." + this.driverId + ".globus-token");
+        String globusToken = getConfigParam(GLOBUS_TOKEN);
+        
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         return accessToken.getOtherTokens().get(0).getAccessToken();
     }
 
+
+
     private void validatePath(String relPath) throws IOException {
         try {
             URI absoluteURI = new URI(baseUrl + "/" + relPath);
@@ -255,7 +260,7 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
 //Fix
         // ToDo - support remote auxiliary Files
         if (auxiliaryTag == null) {
-            String secretKey = System.getProperty("dataverse.files." + this.driverId + ".secret-key");
+            String secretKey = getConfigParam(SECRET_KEY);
             if (secretKey == null) {
                 return baseUrl + "/" + path;
             } else {
@@ -267,12 +272,12 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
     }
 
     public static boolean isDataverseManaged(String driverId) {
-        return Boolean.getBoolean("dataverse.files." + driverId + ".managed");
+        return Boolean.parseBoolean(getConfigParamForDriver(driverId, MANAGED));
     }
 
     static boolean isValidIdentifier(String driverId, String storageId) {
         String baseIdentifier = storageId.substring(storageId.lastIndexOf("//") + 2);
-        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
+        String baseUrl = getConfigParamForDriver(driverId, BASE_URL);
         if (baseUrl == null) {
             return false;
         }
@@ -312,6 +317,14 @@ public String getStorageLocation() throws IOException {
         }
     }
     
+    public static String getEndpointId(String driverId) {
+        String baseUrl = getConfigParamForDriver(driverId, BASE_URL);
+        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf(DataAccess.SEPARATOR) + 3);
+        int pathStart = endpointWithBasePath.indexOf("/");
+        return pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
+        
+    }
+    
     public static void main(String[] args) {
         System.out.println("Running the main method");
         if (args.length > 0) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index 9c1f5ba23aa..aafab038ae2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -64,6 +64,13 @@
 public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
+
+    static final String BASE_URL = "base-url";
+    static final String BASE_STORE = "base-store";
+    static final String SECRET_KEY = "secret-key";
+    static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
+    static final String REMOTE_STORE_NAME = "remote-store-name";
+    static final String REMOTE_STORE_URL = "remote-store-url";
     
     protected StorageIO<DvObject> baseStore = null;
     protected String path = null;
@@ -404,7 +411,7 @@ public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException
 
     @Override
     public boolean downloadRedirectEnabled() {
-        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".download-redirect");
+        String optionValue = getConfigParam(StorageIO.DOWNLOAD_REDIRECT);
         if ("true".equalsIgnoreCase(optionValue)) {
             return true;
         }
@@ -421,7 +428,7 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
 
         // ToDo - support remote auxiliary Files
         if (auxiliaryTag == null) {
-            String secretKey = System.getProperty("dataverse.files." + this.driverId + ".secret-key");
+            String secretKey = getConfigParam(SECRET_KEY);
             if (secretKey == null) {
                 return baseUrl + "/" + path;
             } else {
@@ -434,7 +441,7 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
     }
 
     int getUrlExpirationMinutes() {
-        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".url-expiration-minutes");
+        String optionValue = getConfigParam(URL_EXPIRATION_MINUTES);
         if (optionValue != null) {
             Integer num;
             try {
@@ -450,7 +457,7 @@ int getUrlExpirationMinutes() {
     }
 
     protected void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
-        baseUrl = System.getProperty("dataverse.files." + this.driverId + ".base-url");
+        baseUrl = getConfigParam(BASE_URL);
         if (baseUrl == null) {
             throw new IOException("dataverse.files." + this.driverId + ".base-url is required");
         } else {
@@ -467,7 +474,7 @@ protected void configureStores(DataAccessRequest req, String driverId, String st
         if (baseStore == null) {
             String baseDriverId = getBaseStoreIdFor(driverId);
             String fullStorageLocation = null;
-            String baseDriverType = System.getProperty("dataverse.files." + baseDriverId + ".type",
+            String baseDriverType = getConfigParamForDriver(baseDriverId, StorageIO.TYPE,
                     DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
 
             if (dvObject instanceof Dataset) {
@@ -480,17 +487,17 @@ protected void configureStores(DataAccessRequest req, String driverId, String st
                     switch (baseDriverType) {
                     case DataAccess.S3:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".bucket-name") + "/"
+                                + getConfigParamForDriver(baseDriverId, S3AccessIO.BUCKET_NAME) + "/"
                                 + fullStorageLocation;
                         break;
                     case DataAccess.FILE:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
+                                + getConfigParamForDriver(baseDriverId, FileAccessIO.DIRECTORY, "/tmp/files")
                                 + "/" + fullStorageLocation;
                         break;
                     default:
                         logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
-                                + System.getProperty("dataverse.files." + baseDriverId + ".type"));
+                                + getConfigParamForDriver(baseDriverId, StorageIO.TYPE));
                         throw new IOException("Not supported");
                     }
 
@@ -508,17 +515,17 @@ protected void configureStores(DataAccessRequest req, String driverId, String st
                     switch (baseDriverType) {
                     case DataAccess.S3:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".bucket-name") + "/"
+                                + getConfigParamForDriver(baseDriverId, S3AccessIO.BUCKET_NAME) + "/"
                                 + fullStorageLocation;
                         break;
                     case DataAccess.FILE:
                         fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + System.getProperty("dataverse.files." + baseDriverId + ".directory", "/tmp/files")
+                                + getConfigParamForDriver(baseDriverId, FileAccessIO.DIRECTORY, "/tmp/files")
                                 + "/" + fullStorageLocation;
                         break;
                     default:
                         logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
-                                + System.getProperty("dataverse.files." + baseDriverId + ".type"));
+                                + getConfigParamForDriver(baseDriverId, StorageIO.TYPE));
                         throw new IOException("Not supported");
                     }
                 }
@@ -528,9 +535,9 @@ protected void configureStores(DataAccessRequest req, String driverId, String st
                 ((S3AccessIO<?>) baseStore).setMainDriver(false);
             }
         }
-        remoteStoreName = System.getProperty("dataverse.files." + this.driverId + ".remote-store-name");
+        remoteStoreName = getConfigParam(REMOTE_STORE_NAME);
         try {
-            remoteStoreUrl = new URL(System.getProperty("dataverse.files." + this.driverId + ".remote-store-url"));
+            remoteStoreUrl = new URL(getConfigParam(REMOTE_STORE_URL));
         } catch (MalformedURLException mfue) {
             logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
         }
@@ -623,7 +630,7 @@ public void saveInputStream(InputStream inputStream, Long filesize) throws IOExc
 
     static boolean isValidIdentifier(String driverId, String storageId) {
         String urlPath = storageId.substring(storageId.lastIndexOf("//") + 2);
-        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
+        String baseUrl = getConfigParamForDriver(driverId, BASE_URL);
         try {
             URI absoluteURI = new URI(baseUrl + "/" + urlPath);
             if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
@@ -638,7 +645,7 @@ static boolean isValidIdentifier(String driverId, String storageId) {
     }
 
     public static String getBaseStoreIdFor(String driverId) {
-        return System.getProperty("dataverse.files." + driverId + ".base-store");
+        return getConfigParamForDriver(driverId, BASE_STORE);
     }
 
     @Override
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index b0f9f0ffb05..decc65943ec 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -88,6 +88,16 @@ public class S3AccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Config config = ConfigProvider.getConfig();
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.S3AccessIO");
+    static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
+    static final String CUSTOM_ENDPOINT_URL = "custom-endpoint-url";
+    static final String PROXY_URL = "proxy-url";
+    static final String BUCKET_NAME = "bucket-name";
+    static final String MIN_PART_SIZE = "min-part-size";
+    static final String CUSTOM_ENDPOINT_REGION = "custom-endpoint-region";
+    static final String PATH_STYLE_ACCESS = "path-style-access";
+    static final String PAYLOAD_SIGNING = "payload-signing";
+    static final String CHUNKED_ENCODING = "chunked-encoding";
+    static final String PROFILE = "profile";
     
     private boolean mainDriver = true;
 
@@ -103,8 +113,8 @@ public S3AccessIO(T dvObject, DataAccessRequest req, String driverId) {
             minPartSize = getMinPartSize(driverId);
             s3=getClient(driverId);
             tm=getTransferManager(driverId);
-            endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url", "");
-            proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url", "");
+            endpoint = getConfigParam(CUSTOM_ENDPOINT_URL, "");
+            proxy = getConfigParam(PROXY_URL, "");
             if(!StringUtil.isEmpty(proxy)&&StringUtil.isEmpty(endpoint)) {
                 logger.severe(driverId + " config error: Must specify a custom-endpoint-url if proxy-url is specified");
             }
@@ -842,7 +852,7 @@ private static String getMainFileKey(String baseKey, String storageIdentifier, S
 
     @Override
     public boolean downloadRedirectEnabled() {
-        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".download-redirect");
+        String optionValue = getConfigParam(DOWNLOAD_REDIRECT);
         if ("true".equalsIgnoreCase(optionValue)) {
             return true;
         }
@@ -1066,7 +1076,7 @@ public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String s
     }
     
     int getUrlExpirationMinutes() {
-        String optionValue = System.getProperty("dataverse.files." + this.driverId + ".url-expiration-minutes"); 
+        String optionValue = getConfigParam(URL_EXPIRATION_MINUTES); 
         if (optionValue != null) {
             Integer num; 
             try {
@@ -1082,7 +1092,7 @@ int getUrlExpirationMinutes() {
     }
     
     private static String getBucketName(String driverId) {
-        return System.getProperty("dataverse.files." + driverId + ".bucket-name");
+        return getConfigParamForDriver(driverId, BUCKET_NAME);
     }
     
     private static long getMinPartSize(String driverId) {
@@ -1090,7 +1100,7 @@ private static long getMinPartSize(String driverId) {
         // (minimum allowed is 5*1024**2 but it probably isn't worth the complexity starting at ~5MB. Also -  confirmed that they use base 2 definitions)
         long min = 5 * 1024 * 1024l; 
 
-        String partLength = System.getProperty("dataverse.files." + driverId + ".min-part-size");
+        String partLength = getConfigParamForDriver(driverId, MIN_PART_SIZE);
         try {
             if (partLength != null) {
                 long val = Long.parseLong(partLength);
@@ -1139,12 +1149,12 @@ private static AmazonS3 getClient(String driverId) {
              * Pass in a URL pointing to your S3 compatible storage.
              * For possible values see https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/client/builder/AwsClientBuilder.EndpointConfiguration.html
              */
-            String s3CEUrl = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url", "");
+            String s3CEUrl = getConfigParamForDriver(driverId, CUSTOM_ENDPOINT_URL, "");
             /**
              * Pass in a region to use for SigV4 signing of requests.
              * Defaults to "dataverse" as it is not relevant for custom S3 implementations.
              */
-            String s3CERegion = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-region", "dataverse");
+            String s3CERegion = getConfigParamForDriver(driverId, CUSTOM_ENDPOINT_REGION, "dataverse");
 
             // if the admin has set a system property (see below) we use this endpoint URL instead of the standard ones.
             if (!s3CEUrl.isEmpty()) {
@@ -1154,7 +1164,7 @@ private static AmazonS3 getClient(String driverId) {
              * Pass in a boolean value if path style access should be used within the S3 client.
              * Anything but case-insensitive "true" will lead to value of false, which is default value, too.
              */
-            Boolean s3pathStyleAccess = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".path-style-access", "false"));
+            Boolean s3pathStyleAccess = Boolean.parseBoolean(getConfigParamForDriver(driverId, PATH_STYLE_ACCESS, "false"));
             // some custom S3 implementations require "PathStyleAccess" as they us a path, not a subdomain. default = false
             s3CB.withPathStyleAccessEnabled(s3pathStyleAccess);
 
@@ -1162,12 +1172,12 @@ private static AmazonS3 getClient(String driverId) {
              * Pass in a boolean value if payload signing should be used within the S3 client.
              * Anything but case-insensitive "true" will lead to value of false, which is default value, too.
              */
-            Boolean s3payloadSigning = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".payload-signing","false"));
+            Boolean s3payloadSigning = Boolean.parseBoolean(getConfigParamForDriver(driverId, PAYLOAD_SIGNING,"false"));
             /**
              * Pass in a boolean value if chunked encoding should not be used within the S3 client.
              * Anything but case-insensitive "false" will lead to value of true, which is default value, too.
              */
-            Boolean s3chunkedEncoding = Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".chunked-encoding","true"));
+            Boolean s3chunkedEncoding = Boolean.parseBoolean(getConfigParamForDriver(driverId, CHUNKED_ENCODING,"true"));
             // Openstack SWIFT S3 implementations require "PayloadSigning" set to true. default = false
             s3CB.setPayloadSigningEnabled(s3payloadSigning);
             // Openstack SWIFT S3 implementations require "ChunkedEncoding" set to false. default = true
@@ -1178,7 +1188,7 @@ private static AmazonS3 getClient(String driverId) {
              * Pass in a string value if this storage driver should use a non-default AWS S3 profile.
              * The default is "default" which should work when only one profile exists.
              */
-            String s3profile = System.getProperty("dataverse.files." + driverId + ".profile","default");
+            String s3profile = getConfigParamForDriver(driverId, PROFILE,"default");
             ProfileCredentialsProvider profileCredentials = new ProfileCredentialsProvider(s3profile);
     
             // Try to retrieve credentials via Microprofile Config API, too. For production use, you should not use env
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index f3c2ef5f513..fda7e919a0a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -50,6 +50,13 @@
 
 public abstract class StorageIO<T extends DvObject> {
 
+    static final String INGEST_SIZE_LIMIT = "ingestsizelimit";
+    static final String PUBLIC = "public";
+    static final String TYPE = "type";
+    static final String UPLOAD_REDIRECT = "upload-redirect";
+    static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band";
+    protected static final String DOWNLOAD_REDIRECT = "download-redirect";
+
     public StorageIO() {
 
     }
@@ -572,7 +579,7 @@ protected boolean isWriteAccessRequested(DataAccessOption... options) throws IOE
     }
 
     public boolean isBelowIngestSizeLimit() {
-        long limit = Long.parseLong(System.getProperty("dataverse.files." + this.driverId + ".ingestsizelimit", "-1"));
+        long limit = Long.parseLong(getConfigParam(INGEST_SIZE_LIMIT, "-1"));
         if (limit > 0 && getSize() > limit) {
             return false;
         } else {
@@ -597,7 +604,7 @@ public static boolean isPublicStore(String driverId) {
         // Read once and cache
         if (!driverPublicAccessMap.containsKey(driverId)) {
             driverPublicAccessMap.put(driverId,
-                    Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".public")));
+                    Boolean.parseBoolean(getConfigParamForDriver(driverId, PUBLIC)));
         }
         return driverPublicAccessMap.get(driverId);
     }
@@ -607,9 +614,9 @@ public static String getDriverPrefix(String driverId) {
     }
 
     public static boolean isDirectUploadEnabled(String driverId) {
-        return (System.getProperty("dataverse.files." + driverId + ".type").equals(DataAccess.S3)
-                && Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect")))
-                || Boolean.parseBoolean(System.getProperty("dataverse.files." + driverId + ".upload-out-of-band"));
+        return (getConfigParamForDriver(driverId, TYPE).equals(DataAccess.S3)
+                && Boolean.parseBoolean(getConfigParamForDriver(driverId, UPLOAD_REDIRECT)))
+                || Boolean.parseBoolean(getConfigParamForDriver(driverId, UPLOAD_OUT_OF_BAND));
     }
 
     // Check that storageIdentifier is consistent with store's config
@@ -638,5 +645,30 @@ protected static boolean usesStandardNamePattern(String identifier) {
      * @throws IOException 
      */
     public abstract long retrieveSizeFromMedia() throws IOException;
+    
+    
+    /* Convenience methods to get a driver-specific parameter
+     * 
+     * - with or without a default
+     * - static or per object
+     * 
+     * @param parameterName
+     * @return the parameter value
+     */
+    
+    protected String getConfigParam(String parameterName) {
+        return getConfigParam(parameterName, null);
+    }
+
+    protected String getConfigParam(String parameterName, String defaultValue) {
+        return getConfigParamForDriver(this.driverId, parameterName, defaultValue);
+    }
+
+    protected static String getConfigParamForDriver(String driverId, String parameterName) {
+        return getConfigParamForDriver(driverId, parameterName, null);
+    }
+    protected static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) {
+        return System.getProperty("dataverse.files." + driverId + "." + parameterName, defaultValue);
+    }
 
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
index 0d1dab581fe..105a60ab418 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/SwiftAccessIO.java
@@ -50,6 +50,17 @@ public class SwiftAccessIO<T extends DvObject> extends StorageIO<T> {
     private String swiftLocation; 
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO");
+    private static final String IS_PUBLIC_CONTAINER = "isPublicContainer";
+    private static final String FOLDER_PATH_SEPARATOR = "folderPathSeparator";
+    private static final String DEFAULT_ENDPOINT = "defaultEndpoint";
+    private static final String TEMPORARY_URL_EXPIRY_TIME = "temporaryUrlExpiryTime";
+    private static final String AUTH_URL = "authUrl";
+    private static final String USERNAME = "username";
+    private static final String PASSWORD = "password";
+    private static final String TENANT = "tenant";
+    private static final String AUTH_TYPE = "authType";
+    private static final String HASH_KEY = "hashKey";
+    private static final String ENDPOINT = "endpoint";
 
 	public SwiftAccessIO() {
 		//Partially functional StorageIO object - constructor only for testing
@@ -70,10 +81,10 @@ public SwiftAccessIO(String swiftLocation, String driverId) {
     }
 
     private void readSettings() {
-    	isPublicContainer = Boolean.parseBoolean(System.getProperty("dataverse.files." + this.driverId + ".isPublicContainer", "true"));
-        swiftFolderPathSeparator = System.getProperty("dataverse.files." + this.driverId + ".folderPathSeparator", "_");
-        swiftDefaultEndpoint = System.getProperty("dataverse.files." + this.driverId + ".defaultEndpoint");
-        tempUrlExpires = Integer.parseInt(System.getProperty("dataverse.files." + this.driverId + ".temporaryUrlExpiryTime", "60"));
+    	isPublicContainer = Boolean.parseBoolean(getConfigParam(IS_PUBLIC_CONTAINER, "true"));
+        swiftFolderPathSeparator = getConfigParam(FOLDER_PATH_SEPARATOR, "_");
+        swiftDefaultEndpoint = getConfigParam(DEFAULT_ENDPOINT);
+        tempUrlExpires = Integer.parseInt(getConfigParam(TEMPORARY_URL_EXPIRY_TIME, "60"));
 		
 	}
 
@@ -740,12 +751,12 @@ private StoredObject openSwiftAuxFile(boolean writeAccess, String auxItemTag) th
     }
 
     Account authenticateWithSwift(String swiftEndPoint) throws IOException {
-        String swiftEndPointAuthUrl = System.getProperty("dataverse.files." + this.driverId + ".authUrl." + swiftEndPoint);
-        String swiftEndPointUsername = System.getProperty("dataverse.files." + this.driverId + ".username." + swiftEndPoint);
-        String swiftEndPointSecretKey = System.getProperty("dataverse.files." + this.driverId + ".password." + swiftEndPoint);
-        String swiftEndPointTenantName = System.getProperty("dataverse.files." + this.driverId + ".tenant." + swiftEndPoint);
-        String swiftEndPointAuthMethod = System.getProperty("dataverse.files." + this.driverId + ".authType." + swiftEndPoint);
-        String swiftEndPointTenantId = System.getProperty("dataverse.files." + this.driverId + ".tenant." + swiftEndPoint);
+        String swiftEndPointAuthUrl = getConfigParam(AUTH_URL + "." + swiftEndPoint);
+        String swiftEndPointUsername = getConfigParam(USERNAME + "." + swiftEndPoint);
+        String swiftEndPointSecretKey = getConfigParam(PASSWORD + "." + swiftEndPoint);
+        String swiftEndPointTenantName = getConfigParam(TENANT + "." + swiftEndPoint);
+        String swiftEndPointAuthMethod = getConfigParam(AUTH_TYPE + "." + swiftEndPoint);
+        String swiftEndPointTenantId = getConfigParam(TENANT + "." + swiftEndPoint);
 
         if (swiftEndPointAuthUrl == null || swiftEndPointUsername == null || swiftEndPointSecretKey == null
                 || "".equals(swiftEndPointAuthUrl) || "".equals(swiftEndPointUsername) || "".equals(swiftEndPointSecretKey)) {
@@ -814,7 +825,7 @@ private String getSwiftFileURI(StoredObject fileObject) throws IOException {
     private String hmac = null;
     public String generateTempUrlSignature(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException {
         if (hmac == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) {
-            String secretKey = System.getProperty("dataverse.files." + this.driverId + ".hashKey." + swiftEndPoint);
+            String secretKey = getConfigParam(HASH_KEY + "." + swiftEndPoint);
             if (secretKey == null) {
                 throw new IOException("Please input a hash key under dataverse.files." + this.driverId + ".hashKey." + swiftEndPoint);
             }
@@ -841,7 +852,7 @@ public long generateTempUrlExpiry(int duration, long currentTime) {
 
     private String temporaryUrl = null;
     private String generateTemporarySwiftUrl(String swiftEndPoint, String containerName, String objectName, int duration) throws IOException {
-        String baseUrl = System.getProperty("dataverse.files." + this.driverId + ".endpoint." + swiftEndPoint);
+        String baseUrl = getConfigParam(ENDPOINT + "." + swiftEndPoint);
         String path = "/v1/" + containerName + "/" + objectName;
         
         if (temporaryUrl == null || isExpiryExpired(generateTempUrlExpiry(duration, System.currentTimeMillis()), duration, System.currentTimeMillis())) {

From e22c5b45cc6167fd3cb7f67cf817a7aa3841d321 Mon Sep 17 00:00:00 2001
From: Saikiran Patil <patilrsaikiran@gmail.com>
Date: Fri, 20 Oct 2023 00:52:49 +0530
Subject: [PATCH 606/922] remove extra white spaces

---
 doc/sphinx-guides/source/api/native-api.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index ffb15b41fd1..43d0b75a618 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3108,7 +3108,7 @@ A curl example using an ``ID``
   export ID=24
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false,"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
     "$SERVER_URL/api/files/$ID/metadata"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3116,7 +3116,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false,"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
     "http://demo.dataverse.org/api/files/24/metadata"
 
 A curl example using a ``PERSISTENT_ID``
@@ -3128,7 +3128,7 @@ A curl example using a ``PERSISTENT_ID``
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false,"dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
     "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3136,7 +3136,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","restrict":false, "dataFileTags":["Survey"]}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
     "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
 
 Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.

From 7fa9ce239ce838829bad9d8fd8cfa916390aad60 Mon Sep 17 00:00:00 2001
From: Saikiran Patil <patilrsaikiran@gmail.com>
Date: Fri, 20 Oct 2023 06:56:42 +0530
Subject: [PATCH 607/922] added back "categories" tag in codes

---
 doc/sphinx-guides/source/api/native-api.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 43d0b75a618..24475dfaeb1 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3108,7 +3108,7 @@ A curl example using an ``ID``
   export ID=24
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
     "$SERVER_URL/api/files/$ID/metadata"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3116,7 +3116,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
     "http://demo.dataverse.org/api/files/24/metadata"
 
 A curl example using a ``PERSISTENT_ID``
@@ -3128,7 +3128,7 @@ A curl example using a ``PERSISTENT_ID``
   export PERSISTENT_ID=doi:10.5072/FK2/AAA000
 
   curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
     "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
 
 The fully expanded example above (without environment variables) looks like this:
@@ -3136,7 +3136,7 @@ The fully expanded example above (without environment variables) looks like this
 .. code-block:: bash
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
-    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","dataFileTags":["Survey"],"restrict":false}' \
+    -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
     "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
 
 Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.

From 56192f83b95239dade92a83e6600d4f6be172119 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 20 Oct 2023 10:41:07 -0400
Subject: [PATCH 608/922] remove unused params, send identifiers and paths, try
 scheduler, etc.

---
 .../harvard/iq/dataverse/api/Datasets.java    | 28 +++++++++------
 .../iq/dataverse/dataaccess/DataAccess.java   | 24 +++++++++++++
 .../iq/dataverse/dataaccess/S3AccessIO.java   |  4 +++
 .../iq/dataverse/dataaccess/StorageIO.java    |  5 +++
 .../dataverse/globus/GlobusServiceBean.java   | 36 ++++++++++++-------
 5 files changed, 75 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 73c00c1d21b..b3913c8aee4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3451,15 +3451,25 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         if(!DataAccess.getDriverType(storeId).equals(DataAccess.GLOBUS)) {
             return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled"));
         }
+        
+        URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
+
         boolean managed = GlobusOverlayAccessIO.isDataverseManaged(storeId);
         String endpoint = GlobusOverlayAccessIO.getEndpointId(storeId);
+
         JsonObjectBuilder queryParams = Json.createObjectBuilder();
         queryParams.add("queryParameters",
                 Json.createArrayBuilder().add(Json.createObjectBuilder().add("datasetId", "{datasetId}"))
                         .add(Json.createObjectBuilder().add("siteUrl", "{siteUrl}"))
                         .add(Json.createObjectBuilder().add("datasetVersion", "{datasetVersion}"))
-                        .add(Json.createObjectBuilder().add("dvLocale", "{localeCode}")).add(Json.createObjectBuilder()
-                                .add("datasetPid", "{datasetPid}").add("managed", managed).add("endpoint", endpoint)));
+                        .add(Json.createObjectBuilder().add("dvLocale", "{localeCode}"))
+                        .add(Json.createObjectBuilder().add("datasetPid", "{datasetPid}")));
+        JsonObject substitutedParams = tokenUtil.getParams(queryParams.build());
+        JsonObjectBuilder params = Json.createObjectBuilder();
+        substitutedParams.keySet().forEach((key) -> {
+            params.add(key, substitutedParams.get(key));
+        });
+        params.add("managed", Boolean.toString(managed)).add("endpoint", endpoint);
 
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
@@ -3470,14 +3480,12 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
                 .add(URLTokenUtil.TIMEOUT, 300));
-        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing")
-                .add(URLTokenUtil.HTTP_METHOD, "GET")
-                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
-                .add(URLTokenUtil.TIMEOUT, 300));
-        
+        allowedApiCalls.add(
+                Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
+                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
+                        .add(URLTokenUtil.TIMEOUT, 300));
 
-        URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
-        return ok(tokenUtil.createPostBody(tokenUtil.getParams(queryParams.build()), allowedApiCalls.build()));
+        return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
     }
     
     /** Requests permissions for a given globus user to upload to the dataset
@@ -3544,7 +3552,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
             JsonObject response = globusService.requestAccessiblePaths(principal, dataset, numberOfPaths);
             switch (response.getInt("status")) {
             case 201:
-                return ok(response.getJsonArray("paths"));
+                return ok(response.getJsonObject("paths"));
             case 400:
                 return badRequest("Unable to grant permission");
             case 409:
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
index 4eaa5f9269c..4a4d3f57f83 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
@@ -403,4 +403,28 @@ public static boolean isValidDirectStorageIdentifier(String storageId) {
         }
         return false;
     }
+
+
+
+    public static String getNewStorageIdentifier(String driverId) {
+        String storageType = DataAccess.getDriverType(driverId);
+        if (storageType.equals("tmp") || storageType.equals("Undefined")) {
+            return null;
+        }
+        switch (storageType) {
+        case FILE:
+            return FileAccessIO.getNewIdentifier(driverId);
+        case SWIFT:
+            return SwiftAccessIO.getNewIdentifier(driverId);
+        case S3:
+            return S3AccessIO.getNewIdentifier(driverId);
+        case REMOTE:
+            return RemoteOverlayAccessIO.getNewIdentifier(driverId);
+        case GLOBUS:
+            return GlobusOverlayAccessIO.getNewIdentifier(driverId);
+        default:
+            logger.warning("Request to validate for storage driver: " + driverId);
+        }
+        return null;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index decc65943ec..7fcb2c9dd36 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -1400,4 +1400,8 @@ public long retrieveSizeFromMedia() throws IOException {
         }
         return objectMetadata.getContentLength();
     }
+    
+    public static String getNewIdentifier(String driverId) {
+        return driverId + DataAccess.SEPARATOR + getConfigParamForDriver(driverId, BUCKET_NAME) + ":" + FileUtil.generateStorageIdentifier();
+    }
 }
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index fda7e919a0a..14fc9254c59 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -25,6 +25,7 @@
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.datavariable.DataVariable;
+import edu.harvard.iq.dataverse.util.FileUtil;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -670,5 +671,9 @@ protected static String getConfigParamForDriver(String driverId, String paramete
     protected static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) {
         return System.getProperty("dataverse.files." + driverId + "." + parameterName, defaultValue);
     }
+    
+    public static String getNewIdentifier(String driverId) {
+        return driverId + DataAccess.SEPARATOR + FileUtil.generateStorageIdentifier();
+    }
 
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 2c0edd070f3..35cb748068b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -2,6 +2,7 @@
 
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Scheduler;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.GsonBuilder;
 import edu.harvard.iq.dataverse.*;
@@ -36,6 +37,7 @@
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
+import java.util.Map.Entry;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
@@ -257,12 +259,16 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
                     permissions.setId(globusResponse.getString("access_id"));
                     monitorTemporaryPermissions(permissions.getId(), dataset.getId());
                     logger.info("Access rule " + permissions.getId() + " was created successfully");
-                    JsonArrayBuilder pathArray = Json.createArrayBuilder();
+                    
+                    String driverId = dataset.getEffectiveStorageDriverId();
+                    JsonObjectBuilder paths = Json.createObjectBuilder();
                     for(int i=0;i<numberOfPaths;i++) {
-                        pathArray.add(getUniqueFilePath(endpoint));
+                        String storageIdentifier = DataAccess.getNewStorageIdentifier(driverId);
+                        int lastIndex = Math.max(storageIdentifier.lastIndexOf("/"), storageIdentifier.lastIndexOf(":")); 
+                        paths.add(storageIdentifier, endpoint.getBasePath() + "/" + storageIdentifier.substring(lastIndex + 1));
                     
                     }
-                    response.add("paths", pathArray.build());
+                    response.add("paths", paths.build());
                     
                 } else {
                     //Shouldn't happen!
@@ -277,18 +283,23 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
             return response.build();
     }
 
-    private String getUniqueFilePath(GlobusEndpoint endpoint) {
+    private Entry<String,String> getUniqueFilePath(GlobusEndpoint endpoint) {
         // TODO See if generated identifier exists at globus endpoint
-        return endpoint.getBasePath() + "/" + FileUtil.generateStorageIdentifier();
+        String sid=FileUtil.generateStorageIdentifier();
+        String path = endpoint.getBasePath() + "/" + FileUtil.generateStorageIdentifier();
+        return null;
     }
 
     //Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
-            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
+//            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
+            .expireAfterWrite(Duration.of(1, ChronoUnit.MINUTES))
+            .scheduler(Scheduler.systemScheduler())
             .evictionListener((ruleId, datasetId, cause) -> {
                 //Delete rules that expire
+                logger.info("Rule " + ruleId + " expired");
                 Dataset dataset = datasetSvc.find(datasetId);
-                deletePermission((String) ruleId, dataset, null);
+                deletePermission((String) ruleId, dataset, logger);
               })
             
             .build();
@@ -583,7 +594,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
         String appUrl;
         if (upload) {
             appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
-                    + "/upload?datasetPid={datasetPid}&siteUrl={siteUrl}&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}";
+                    + "/upload?dvLocale={localeCode}";
             String callback = SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + d.getId()
                     + "/globusUploadParameters?locale=" + localeCode;
             if (apiToken != null) {
@@ -611,7 +622,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
                         + rawStorageId + "&fileName=" + df.getCurrentName();
             }
         }
-        String finalUrl = tokenUtil.replaceTokensWithValues(appUrl) + "&storePrefix=" + storePrefix;
+        String finalUrl = tokenUtil.replaceTokensWithValues(appUrl);
         logger.info("Calling app: " + finalUrl);
         return finalUrl;
     }
@@ -827,9 +838,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                 globusLogger.info("Files failures: " + countError.toString());
                 globusLogger.info("Finished upload via Globus job.");
 
-                if (fileHandlerSuceeded) {
-                    fileHandler.close();
-                }
+
 
             } catch (Exception e) {
                 logger.info("Exception from globusUpload call ");
@@ -842,6 +851,9 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
             deletePermission(ruleId, dataset, globusLogger);
             globusLogger.info("Removed upload permission: " + ruleId);
         }
+        if (fileHandlerSuceeded) {
+            fileHandler.close();
+        }
     }
 
     public String addFilesAsync(String curlCommand, Logger globusLogger)

From ec119a4afe29ca6731fc61a940a1d75930ccc129 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 20 Oct 2023 15:12:46 -0400
Subject: [PATCH 609/922] initial changes to allow xfer to S3 store

---
 .../harvard/iq/dataverse/api/Datasets.java    | 17 ++++++---
 .../dataaccess/GlobusAccessibleStore.java     | 36 ++++++++++++++++++
 .../dataaccess/GlobusOverlayAccessIO.java     | 31 +++++----------
 .../iq/dataverse/dataaccess/S3AccessIO.java   |  2 +-
 .../dataverse/globus/GlobusServiceBean.java   | 38 +++++++------------
 5 files changed, 71 insertions(+), 53 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b3913c8aee4..1cfd4e3bef0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -65,7 +65,7 @@
 import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
 import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
-import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.S3AccessIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
@@ -3448,14 +3448,15 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
             return wr.getResponse();
         }
         String storeId = dataset.getEffectiveStorageDriverId();
-        if(!DataAccess.getDriverType(storeId).equals(DataAccess.GLOBUS)) {
+        //acceptsGlobusTransfers should only be true for an S3 or globus store
+        if(!GlobusAccessibleStore.acceptsGlobusTransfers(storeId) && !GlobusAccessibleStore.allowsGlobusReferences(storeId)) {
             return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled"));
         }
         
         URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
 
-        boolean managed = GlobusOverlayAccessIO.isDataverseManaged(storeId);
-        String endpoint = GlobusOverlayAccessIO.getEndpointId(storeId);
+        boolean managed = GlobusAccessibleStore.isDataverseManaged(storeId);
+        String transferEndpoint = GlobusAccessibleStore.getEndpointId(storeId);
 
         JsonObjectBuilder queryParams = Json.createObjectBuilder();
         queryParams.add("queryParameters",
@@ -3469,7 +3470,11 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         substitutedParams.keySet().forEach((key) -> {
             params.add(key, substitutedParams.get(key));
         });
-        params.add("managed", Boolean.toString(managed)).add("endpoint", endpoint);
+        if(transferEndpoint!= null) {
+            params.add("managed", Boolean.toString(managed)).add("endpoint", transferEndpoint);
+        } else {
+            //ToDO: Reference endpoints
+        }
 
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
@@ -3535,7 +3540,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
             return wr.getResponse();
         }
 
-        if(!GlobusOverlayAccessIO.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
+        if(!GlobusAccessibleStore.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
             return badRequest("This dataset does not have managed Globus storage");
         }
             
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
new file mode 100644
index 00000000000..ca029947462
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -0,0 +1,36 @@
+package edu.harvard.iq.dataverse.dataaccess;
+
+public interface GlobusAccessibleStore {
+
+    static final String MANAGED = "managed";
+    static final String GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH = "globus-transfer-endpoint-with-basepath";
+    static final String GLOBUS_REFERENCE_ENDPOINTS_WITH_BASEPATHS = "globus-reference-endpoints-with-basepaths";
+    static final String GLOBUS_TOKEN = "globus-token";
+    
+    public static boolean isDataverseManaged(String driverId) {
+        return Boolean.parseBoolean(StorageIO.getConfigParamForDriver(driverId, MANAGED));
+    }
+    
+    public static String getEndpointId(String driverId) {
+        String baseUrl = StorageIO.getConfigParamForDriver(driverId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
+        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf(DataAccess.SEPARATOR) + 3);
+        int pathStart = endpointWithBasePath.indexOf("/");
+        return pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
+        
+    }
+    
+    public static boolean acceptsGlobusTransfers(String storeId) {
+        if(StorageIO.getConfigParamForDriver(storeId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH) != null) {
+            return true;
+        }
+        return false;
+    }
+
+    public static boolean allowsGlobusReferences(String storeId) {
+        if(StorageIO.getConfigParamForDriver(storeId, GLOBUS_REFERENCE_ENDPOINTS_WITH_BASEPATHS) != null) {
+            return true;
+        }
+        return false;
+    }
+    
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 502340feb7c..97884bd5722 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -42,13 +42,9 @@
  * baseUrl: globus://<globusEndpointId/basePath>
  * 
  */
-public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> {
-
+public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> implements GlobusAccessibleStore {
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
-    static final String GLOBUS_TOKEN = "globus-token";
-    static final String MANAGED = "managed";
-
     /*
      * If this is set to true, the store supports Globus transfer in and
      * Dataverse/the globus app manage file locations, access controls, deletion,
@@ -66,7 +62,7 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
 
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
-        dataverseManaged = isDataverseManaged(this.driverId);
+        dataverseManaged = GlobusAccessibleStore.isDataverseManaged(this.driverId);
     }
 
     private void parsePath() {
@@ -102,7 +98,7 @@ private void parsePath() {
     public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
         this.driverId = driverId;
         configureStores(null, driverId, storageLocation);
-        this.dataverseManaged = isDataverseManaged(this.driverId);
+        this.dataverseManaged = GlobusAccessibleStore.isDataverseManaged(this.driverId);
         if (dataverseManaged) {
             String[] parts = DataAccess.getDriverIdAndStorageLocation(storageLocation);
             path = parts[1];
@@ -115,7 +111,7 @@ public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOE
     }
     
     private String retrieveGlobusAccessToken() {
-        String globusToken = getConfigParam(GLOBUS_TOKEN);
+        String globusToken = getConfigParam(GlobusAccessibleStore.GLOBUS_TOKEN);
         
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
@@ -179,7 +175,11 @@ public long retrieveSizeFromMedia() {
     
     @Override
     public InputStream getInputStream() throws IOException {
-        throw new IOException("Not implemented");
+        if(Boolean.parseBoolean(getConfigParam("endpoint-maps-to-base-store"))) {
+            return baseStore.getInputStream();
+        } else {
+            throw new IOException("Not implemented");
+        }
     }
     
     @Override
@@ -271,10 +271,6 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
         }
     }
 
-    public static boolean isDataverseManaged(String driverId) {
-        return Boolean.parseBoolean(getConfigParamForDriver(driverId, MANAGED));
-    }
-
     static boolean isValidIdentifier(String driverId, String storageId) {
         String baseIdentifier = storageId.substring(storageId.lastIndexOf("//") + 2);
         String baseUrl = getConfigParamForDriver(driverId, BASE_URL);
@@ -283,7 +279,7 @@ static boolean isValidIdentifier(String driverId, String storageId) {
         }
         // Internally managed endpoints require standard name pattern (submitted via
         // /addFile(s) api)
-        if (isDataverseManaged(driverId)) {
+        if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
             boolean hasStandardName = usesStandardNamePattern(baseIdentifier);
             if (hasStandardName) {
                 return true;
@@ -317,13 +313,6 @@ public String getStorageLocation() throws IOException {
         }
     }
     
-    public static String getEndpointId(String driverId) {
-        String baseUrl = getConfigParamForDriver(driverId, BASE_URL);
-        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf(DataAccess.SEPARATOR) + 3);
-        int pathStart = endpointWithBasePath.indexOf("/");
-        return pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
-        
-    }
     
     public static void main(String[] args) {
         System.out.println("Running the main method");
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index 7fcb2c9dd36..6cf19feda86 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -84,7 +84,7 @@
 /* 
     Amazon AWS S3 driver
  */
-public class S3AccessIO<T extends DvObject> extends StorageIO<T> {
+public class S3AccessIO<T extends DvObject> extends StorageIO<T> implements GlobusAccessibleStore {
 
     private static final Config config = ConfigProvider.getConfig();
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.S3AccessIO");
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 35cb748068b..558125e90d1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -37,7 +37,6 @@
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
-import java.util.Map.Entry;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
@@ -56,6 +55,7 @@
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
@@ -283,17 +283,9 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
             return response.build();
     }
 
-    private Entry<String,String> getUniqueFilePath(GlobusEndpoint endpoint) {
-        // TODO See if generated identifier exists at globus endpoint
-        String sid=FileUtil.generateStorageIdentifier();
-        String path = endpoint.getBasePath() + "/" + FileUtil.generateStorageIdentifier();
-        return null;
-    }
-
     //Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
-//            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
-            .expireAfterWrite(Duration.of(1, ChronoUnit.MINUTES))
+            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
             .scheduler(Scheduler.systemScheduler())
             .evictionListener((ruleId, datasetId, cause) -> {
                 //Delete rules that expire
@@ -306,6 +298,7 @@ private Entry<String,String> getUniqueFilePath(GlobusEndpoint endpoint) {
     
     
     private void monitorTemporaryPermissions(String ruleId, long datasetId) {
+        logger.info("Adding rule " + ruleId + " for dataset " + datasetId);
         rulesCache.put(ruleId, datasetId);
     }
 
@@ -519,6 +512,7 @@ static class MakeRequestResponse {
 
     }
 
+    /* unused - may be needed for S3 case
     private MakeRequestResponse findDirectory(String directory, String clientToken, String globusEndpoint)
             throws MalformedURLException {
         URL url = new URL(" https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/ls?path="
@@ -530,7 +524,8 @@ private MakeRequestResponse findDirectory(String directory, String clientToken,
 
         return result;
     }
-
+*/
+    
     /*
     public boolean giveGlobusPublicPermissions(Dataset dataset)
             throws UnsupportedEncodingException, MalformedURLException {
@@ -582,10 +577,8 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
                 apiToken = authSvc.generateApiTokenForUser((AuthenticatedUser) user);
             }
         }
-        String storePrefix = "";
         String driverId = d.getEffectiveStorageDriverId();
         try {
-            storePrefix = DataAccess.getDriverPrefix(driverId);
         } catch (Exception e) {
             logger.warning("GlobusAppUrlForDataset: Failed to get storePrefix for " + driverId);
         }
@@ -765,13 +758,10 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                         countAll++;
                         String storageIdentifier = fileJsonObject.getString("storageIdentifier");
                         String fileName = fileJsonObject.getString("fileName");
-                        String directoryLabel = fileJsonObject.getString("directoryLabel");
                         String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier);
                         //If this is an S3 store, we need to split out the bucket name
                         String[] bits = parts[1].split(":");
-                        String bucketName = "";
                         if(bits.length > 1) {
-                            bucketName = bits[0];
                         }
                         String fileId = bits[bits.length - 1];
                         
@@ -1070,7 +1060,7 @@ public JsonObject calculateMissingMetadataFields(List<String> inputList, Logger
                     .collect(Collectors.toList());
         });
 
-        CompletableFuture completableFuture = allCompletableFuture.thenApply(files -> {
+        CompletableFuture<?> completableFuture = allCompletableFuture.thenApply(files -> {
             return files.stream().map(d -> json(d)).collect(toJsonArray());
         });
 
@@ -1361,15 +1351,17 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         }
         String driverId = dataset.getEffectiveStorageDriverId();
         GlobusEndpoint endpoint = null;
-        String baseUrl = System.getProperty("dataverse.files." + driverId + ".base-url");
+        
+        //ToDo - consolidate with GlobusOverlayAccessIO.parsePath()
+        String baseUrl = System.getProperty("dataverse.files." + driverId + "." + GlobusAccessibleStore.GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
 
         String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
         int pathStart = endpointWithBasePath.indexOf("/");
         logger.info("endpointWithBasePath: " + endpointWithBasePath);
-        String directoryPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
+        String directoryPath = (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "");
         logger.info("directoryPath: " + directoryPath);
 
-        if (GlobusOverlayAccessIO.isDataverseManaged(driverId) && (dataset!=null)) {
+        if (GlobusAccessibleStore.isDataverseManaged(driverId) && (dataset!=null)) {
             directoryPath = directoryPath + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
             logger.info("directoryPath now: " + directoryPath);
 
@@ -1387,7 +1379,7 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         
         logger.info("endpointId: " + endpointId);
         
-        String globusToken = System.getProperty("dataverse.files." + driverId + ".globus-token");
+        String globusToken = System.getProperty("dataverse.files." + driverId + "." + GlobusAccessibleStore.GLOBUS_TOKEN);
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         String clientToken = accessToken.getOtherTokens().get(0).getAccessToken();
@@ -1397,8 +1389,4 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         return endpoint;
     }
     
-    private static boolean isDataverseManaged(String driverId) {
-        return Boolean.getBoolean("dataverse.files." + driverId + ".managed");
-    }
-    
 }

From bfa4756921d42fb72de34a0b238ae0a0632b113a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 20 Oct 2023 15:54:55 -0400
Subject: [PATCH 610/922] remove old settings, use store specific creds to
 monitor task

---
 .../source/developers/big-data-support.rst      |  2 +-
 .../source/installation/config.rst              | 17 +----------------
 .../harvard/iq/dataverse/SettingsWrapper.java   |  6 ++----
 .../edu/harvard/iq/dataverse/api/Access.java    |  5 +++--
 .../dataverse/api/DownloadInstanceWriter.java   |  5 ++---
 .../iq/dataverse/globus/GlobusServiceBean.java  | 17 ++++++++---------
 .../dataverse/settings/SettingsServiceBean.java | 12 ------------
 .../harvard/iq/dataverse/util/SystemConfig.java |  5 -----
 8 files changed, 17 insertions(+), 52 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
index 04885571a01..1917967b3f3 100644
--- a/doc/sphinx-guides/source/developers/big-data-support.rst
+++ b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -168,7 +168,7 @@ As described in that document, Globus transfers can be initiated by choosing the
 
 An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video <https://youtu.be/3ek7F_Dxcjk?t=5289>`_ around the 1 hr 28 min mark.
 
-See also :ref:`Globus settings <:GlobusBasicToken>`.
+See also :ref:`Globus settings <:GlobusSettings>`.
 
 Data Capture Module (DCM)
 -------------------------
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f4e66262ef1..ae0596c1e50 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -3940,22 +3940,7 @@ The URL of an LDN Inbox to which the LDN Announce workflow step will send messag
 
 The list of parent dataset field names for which the LDN Announce workflow step should send messages. See :doc:`/developers/workflows` for details.
 
-.. _:GlobusBasicToken:
-
-:GlobusBasicToken
-+++++++++++++++++
-
-GlobusBasicToken encodes credentials for Globus integration. See :ref:`globus-support` for details.
-
-:GlobusEndpoint
-+++++++++++++++
-
-GlobusEndpoint is Globus endpoint id used with Globus integration. See :ref:`globus-support` for details.
-
-:GlobusStores
-+++++++++++++
-
-A comma-separated list of the S3 stores that are configured to support Globus integration. See :ref:`globus-support` for details.
+.. _:GlobusSettings:
 
 :GlobusAppUrl
 +++++++++++++
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index 0a1d0effc03..8b7f732d03f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -6,6 +6,7 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.Setting;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -333,10 +334,7 @@ public boolean isGlobusFileDownload() {
     }
     
     public boolean isGlobusEnabledStorageDriver(String driverId) {
-        if (globusStoreList == null) {
-            globusStoreList = systemConfig.getGlobusStoresList();
-        }
-        return globusStoreList.contains(driverId);
+        return (GlobusAccessibleStore.acceptsGlobusTransfers(driverId) || GlobusAccessibleStore.allowsGlobusReferences(driverId));
     }
     
     public String getGlobusAppUrl() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index 1aa3f4ffde6..b0129d0ad1e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -46,6 +46,7 @@
 import edu.harvard.iq.dataverse.dataaccess.DataAccessRequest;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.dataaccess.DataFileZipper;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.dataaccess.OptionalAccessService;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.datavariable.DataVariable;
@@ -327,8 +328,8 @@ public Response datafile(@Context ContainerRequestContext crc, @PathParam("fileI
             dInfo.addServiceAvailable(new OptionalAccessService("preprocessed", "application/json", "format=prep", "Preprocessed data in JSON"));
             dInfo.addServiceAvailable(new OptionalAccessService("subset", "text/tab-separated-values", "variables=&lt;LIST&gt;", "Column-wise Subsetting"));
         }
-        
-        if(systemConfig.isGlobusFileDownload() && systemConfig.getGlobusStoresList().contains(DataAccess.getStorageDriverFromIdentifier(df.getStorageIdentifier()))) {
+        String driverId = DataAccess.getStorageDriverFromIdentifier(df.getStorageIdentifier());
+        if(systemConfig.isGlobusFileDownload() && (GlobusAccessibleStore.acceptsGlobusTransfers(driverId) || GlobusAccessibleStore.allowsGlobusReferences(driverId))) {
             dInfo.addServiceAvailable(new OptionalAccessService("GlobusTransfer", df.getContentType(), "format=GlobusTransfer", "Download via Globus"));
         }
         
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
index af681234e82..d645dc3307f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
@@ -206,9 +206,8 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
                         redirect_url_str = null;
                     }
                 }
-                
-                if (systemConfig.isGlobusFileDownload() && systemConfig.getGlobusStoresList()
-                        .contains(DataAccess.getStorageDriverFromIdentifier(dataFile.getStorageIdentifier()))) {
+                String driverId = DataAccess.getStorageDriverFromIdentifier(dataFile.getStorageIdentifier());
+                if (systemConfig.isGlobusFileDownload() && (GlobusAccessibleStore.acceptsGlobusTransfers(driverId) || GlobusAccessibleStore.allowsGlobusReferences(driverId))) {
                     if (di.getConversionParam() != null) {
                         if (di.getConversionParam().equals("format")) {
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 558125e90d1..4f65eb5d316 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -317,12 +317,12 @@ public boolean getSuccessfulTransfers(AccessToken clientTokenUser, String taskId
         return false;
     }
 
-    public GlobusTask getTask(AccessToken clientTokenUser, String taskId, Logger globusLogger) throws MalformedURLException {
+    public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger) throws MalformedURLException {
 
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId);
 
         MakeRequestResponse result = makeRequest(url, "Bearer",
-                clientTokenUser.getOtherTokens().get(0).getAccessToken(), "GET", null);
+                accessToken, "GET", null);
 
         GlobusTask task = null;
 
@@ -661,13 +661,14 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
 
         String taskIdentifier = jsonData.getString("taskIdentifier");
 
+        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
         // globus task status check
-        GlobusTask task = globusStatusCheck(taskIdentifier, globusLogger);
+        GlobusTask task = globusStatusCheck(endpoint, taskIdentifier, globusLogger);
         String taskStatus = getTaskStatus(task);
 
         globusLogger.info("Starting an globusUpload ");
 
-        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+
         String ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
         logger.info("Found rule: " + ruleId);
         if (ruleId != null) {
@@ -947,7 +948,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         }
 
         // globus task status check
-        GlobusTask task = globusStatusCheck(taskIdentifier, globusLogger);
+        GlobusTask task = globusStatusCheck(null, taskIdentifier, globusLogger);
         String taskStatus = getTaskStatus(task);
 
         if (ruleId.length() > 0) {
@@ -976,7 +977,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
 
     Executor executor = Executors.newFixedThreadPool(10);
 
-    private GlobusTask globusStatusCheck(String taskId, Logger globusLogger) throws MalformedURLException {
+    private GlobusTask globusStatusCheck(GlobusEndpoint endpoint, String taskId, Logger globusLogger) throws MalformedURLException {
         boolean taskCompletion = false;
         String status = "";
         GlobusTask task = null;
@@ -985,9 +986,7 @@ private GlobusTask globusStatusCheck(String taskId, Logger globusLogger) throws
             try {
                 globusLogger.info("checking globus transfer task   " + taskId);
                 Thread.sleep(pollingInterval * 1000);
-                AccessToken clientTokenUser = getClientToken(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, ""));
-                // success = globusServiceBean.getSuccessfulTransfers(clientTokenUser, taskId);
-                task = getTask(clientTokenUser, taskId, globusLogger);
+                task = getTask(endpoint.getClientToken(), taskId, globusLogger);
                 if (task != null) {
                     status = task.getStatus();
                     if (status != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
index 0aa403a5116..f2b52ec9533 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
@@ -464,18 +464,6 @@ Whether Harvesting (OAI) service is enabled
          */
         ExportInstallationAsDistributorOnlyWhenNotSet,
 
-        /**
-         * Basic Globus Token for Globus Application
-         */
-        GlobusBasicToken,
-        /**
-         * GlobusEndpoint is Globus endpoint for Globus application
-         */
-        GlobusEndpoint,
-        /** 
-         * Comma separated list of Globus enabled stores
-         */
-        GlobusStores,
         /** Globus App URL
          * 
          */
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 079cbaa999d..e40f55fedd8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -948,11 +948,6 @@ public boolean isGlobusFileDownload() {
         return (isGlobusDownload() && settingsService.isTrueForKey(SettingsServiceBean.Key.GlobusSingleFileTransfer, false));
     }
 
-    public List<String> getGlobusStoresList() {
-    String globusStores = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusStores, "");
-    return Arrays.asList(globusStores.split("\\s*,\\s*"));
-    }
-
     private Boolean getMethodAvailable(String method, boolean upload) {
         String methods = settingsService.getValueForKey(
                 upload ? SettingsServiceBean.Key.UploadMethods : SettingsServiceBean.Key.DownloadMethods);

From 77c606fcabb4ae1434e1fde07557ed3736bb28eb Mon Sep 17 00:00:00 2001
From: Abhinav Rana <142827270+AR-2910@users.noreply.github.com>
Date: Sun, 22 Oct 2023 23:02:41 +0530
Subject: [PATCH 611/922] Update S3AccessIO.java

Issue: Problem with long file stores ID in JVM options #8312

Change made in line 194.
---
 .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index 822ada0b83e..e3c6bfede7c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -191,7 +191,7 @@ public void open(DataAccessOption... options) throws IOException {
                     }
                 } // else we're OK (assumes bucket name in storageidentifier matches the driver's bucketname)
             } else {
-                if(!storageIdentifier.substring((this.driverId + DataAccess.SEPARATOR).length()).contains(":")) {
+                if(!storageIdentifier.contains(":")) {
                     //No driver id or bucket 
                     newStorageIdentifier= this.driverId + DataAccess.SEPARATOR + bucketName + ":" + storageIdentifier;
                 } else {
@@ -1385,4 +1385,4 @@ public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOE
         }
         return toDelete;
     }
-}
\ No newline at end of file
+}

From ea6644b3d6dd2fa2eb1cf7a17f620f7a25a5b871 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 24 Oct 2023 13:11:42 +0100
Subject: [PATCH 612/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFileMetadataCount using JPA Criteria (pending accessStatus search criteria
 filtering)

---
 .../DatasetVersionFilesServiceBean.java       | 44 +++++++++++++++++--
 1 file changed, 40 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index fc662ee80bb..d65014e62a3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -24,6 +24,7 @@
 import java.io.Serializable;
 import java.sql.Timestamp;
 import java.time.LocalDate;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,6 +32,7 @@
 import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
 
 import edu.harvard.iq.dataverse.FileSearchCriteria.FileAccessStatus;
+import jakarta.persistence.criteria.*;
 
 @Stateless
 @Named
@@ -73,10 +75,13 @@ public enum FileDownloadSizeMode {
      * @return long value of total file metadata count
      */
     public long getFileMetadataCount(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        return baseQuery.stream().count();
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
+        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
+        criteriaQuery.select(criteriaBuilder.count(fileMetadataRoot)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate));
+        return em.createQuery(criteriaQuery).getSingleResult();
     }
 
     /**
@@ -254,6 +259,37 @@ private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAcces
         return accessStatusExpression;
     }
 
+    private Predicate createSearchCriteriaPredicate(FileSearchCriteria searchCriteria, CriteriaBuilder criteriaBuilder, CriteriaQuery<Long> criteriaQuery, Root<FileMetadata> fileMetadataRoot) {
+        List<Predicate> predicates = new ArrayList<>();
+        String contentType = searchCriteria.getContentType();
+        if (contentType != null) {
+            predicates.add(criteriaBuilder.equal(fileMetadataRoot.get("dataFile").<String>get("contentType"), contentType));
+        }
+        FileAccessStatus accessStatus = searchCriteria.getAccessStatus();
+        if (accessStatus != null) {
+            // TODO
+        }
+        String categoryName = searchCriteria.getCategoryName();
+        if (categoryName != null) {
+            Root<DataFileCategory> dataFileCategoryRoot = criteriaQuery.from(DataFileCategory.class);
+            predicates.add(criteriaBuilder.equal(dataFileCategoryRoot.get("name"), categoryName));
+            predicates.add(dataFileCategoryRoot.in(fileMetadataRoot.get("fileCategories")));
+        }
+        String tabularTagName = searchCriteria.getTabularTagName();
+        if (tabularTagName != null) {
+            Root<DataFileTag> dataFileTagRoot = criteriaQuery.from(DataFileTag.class);
+            predicates.add(criteriaBuilder.equal(dataFileTagRoot.get("type"), TagLabelToTypes.get(tabularTagName)));
+            predicates.add(dataFileTagRoot.in(fileMetadataRoot.get("dataFile").get("dataFileTags")));
+        }
+        String searchText = searchCriteria.getSearchText();
+        if (searchText != null && !searchText.isEmpty()) {
+            searchText = searchText.trim().toLowerCase();
+            predicates.add(criteriaBuilder.like(fileMetadataRoot.get("label"), "%" + searchText + "%"));
+        }
+        return criteriaBuilder.and(predicates.toArray(new Predicate[]{}));
+    }
+
+    @Deprecated
     private void applyFileSearchCriteriaToQuery(JPAQuery<?> baseQuery, FileSearchCriteria searchCriteria) {
         String contentType = searchCriteria.getContentType();
         if (contentType != null) {

From 65bed77a39eb99a97e9cbdc01e79c3310664898c Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 24 Oct 2023 18:05:50 +0100
Subject: [PATCH 613/922] Added: accessStatus search criteria filtering for JPA
 Criteria queries

---
 .../DatasetVersionFilesServiceBean.java       | 30 +++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index d65014e62a3..c2efa72579f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -24,10 +24,7 @@
 import java.io.Serializable;
 import java.sql.Timestamp;
 import java.time.LocalDate;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
 
@@ -235,6 +232,29 @@ private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion da
         return baseQuery;
     }
 
+    private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus accessStatus, CriteriaBuilder criteriaBuilder, Root<FileMetadata> fileMetadataRoot) {
+        Path<Object> dataFile = fileMetadataRoot.get("dataFile");
+
+        Path<Object> embargo = dataFile.get("embargo");
+        Predicate activelyEmbargoedPredicate = criteriaBuilder.greaterThanOrEqualTo(embargo.<Date>get("dateAvailable"), criteriaBuilder.currentDate());
+        Predicate inactivelyEmbargoedPredicate = criteriaBuilder.isNull(embargo);
+
+        Path<Boolean> isRestricted = dataFile.get("restricted");
+        Predicate isRestrictedPredicate = criteriaBuilder.isTrue(isRestricted);
+        Predicate isUnrestrictedPredicate = criteriaBuilder.isFalse(isRestricted);
+
+        return switch (accessStatus) {
+            case EmbargoedThenRestricted ->
+                    criteriaBuilder.and(activelyEmbargoedPredicate, isRestrictedPredicate);
+            case EmbargoedThenPublic ->
+                    criteriaBuilder.and(activelyEmbargoedPredicate, isUnrestrictedPredicate);
+            case Restricted ->
+                    criteriaBuilder.and(inactivelyEmbargoedPredicate, isRestrictedPredicate);
+            case Public -> criteriaBuilder.and(inactivelyEmbargoedPredicate, isUnrestrictedPredicate);
+        };
+    }
+
+    @Deprecated
     private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAccessStatus accessStatus) {
         QEmbargo embargo = fileMetadata.dataFile.embargo;
         BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
@@ -267,7 +287,7 @@ private Predicate createSearchCriteriaPredicate(FileSearchCriteria searchCriteri
         }
         FileAccessStatus accessStatus = searchCriteria.getAccessStatus();
         if (accessStatus != null) {
-            // TODO
+            predicates.add(createSearchCriteriaAccessStatusPredicate(accessStatus, criteriaBuilder, fileMetadataRoot));
         }
         String categoryName = searchCriteria.getCategoryName();
         if (categoryName != null) {

From a8afef3bf8519b60fe70c84bc071143031b73196 Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Tue, 24 Oct 2023 13:25:02 -0400
Subject: [PATCH 614/922] #10030 document localhost-only behavior of Solr-9.3.0
 and later

---
 .../source/_static/installation/files/etc/init.d/solr       | 2 +-
 .../_static/installation/files/etc/systemd/solr.service     | 2 +-
 doc/sphinx-guides/source/installation/prerequisites.rst     | 6 ++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
index 9cf8902eb14..f7dba504e70 100755
--- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
@@ -7,7 +7,7 @@
 
 SOLR_DIR="/usr/local/solr/solr-9.3.0"
 SOLR_COMMAND="bin/solr"
-SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1"
+SOLR_ARGS="-m 1g"
 SOLR_USER=solr
 
 case $1 in
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
index 0b8a8528490..2ceeb0016d6 100644
--- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
@@ -6,7 +6,7 @@ After = syslog.target network.target remote-fs.target nss-lookup.target
 User = solr
 Type = forking
 WorkingDirectory = /usr/local/solr/solr-9.3.0
-ExecStart = /usr/local/solr/solr-9.3.0/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
+ExecStart = /usr/local/solr/solr-9.3.0/bin/solr start -m 1g
 ExecStop = /usr/local/solr/solr-9.3.0/bin/solr stop
 LimitNOFILE=65000
 LimitNPROC=65000
diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst
index 1847f1b8f63..cff20ccab1d 100644
--- a/doc/sphinx-guides/source/installation/prerequisites.rst
+++ b/doc/sphinx-guides/source/installation/prerequisites.rst
@@ -233,11 +233,9 @@ For systems using init.d (like CentOS 6), download this :download:`Solr init scr
 Securing Solr
 =============
 
-Our sample init script and systemd service file linked above tell Solr to only listen on localhost (127.0.0.1). We strongly recommend that you also use a firewall to block access to the Solr port (8983) from outside networks, for added redundancy.
+As of version 9.3.0, Solr listens solely on localhost for security reasons. If your installation will run Solr on its own host, you will need to edit ``bin/solr.in.sh``, setting ``JETTY_HOST`` to the external IP address of your Solr server to tell Solr to accept external connections.
 
-It is **very important** not to allow direct access to the Solr API from outside networks! Otherwise, any host that can reach the Solr port (8983 by default) can add or delete data, search unpublished data, and even reconfigure Solr. For more information, please see https://lucene.apache.org/solr/guide/7_3/securing-solr.html. A particularly serious security issue that has been identified recently allows a potential intruder to remotely execute arbitrary code on the system. See `RCE in Solr via Velocity Template <https://github.com/veracode-research/solr-injection#7-cve-2019-xxxx-rce-via-velocity-template-by-_s00py>`_ for more information.
-
-If you're running your Dataverse installation across multiple service hosts you'll want to remove the jetty.host argument (``-j jetty.host=127.0.0.1``) from the startup command line, but make sure Solr is behind a firewall and only accessible by the Dataverse installation host(s), by specific ip address(es).
+We strongly recommend that you also use a firewall to block access to the Solr port (8983) from outside networks. It is **very important** not to allow direct access to the Solr API from outside networks! Otherwise, any host that can reach Solr can add or delete data, search unpublished data, and even reconfigure Solr. For more information, please see https://solr.apache.org/guide/solr/latest/deployment-guide/securing-solr.html
 
 We additionally recommend that the Solr service account's shell be disabled, as it isn't necessary for daily operation::
 

From 68081a936fb927cbd20f0e9eb9630a3752121a13 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 24 Oct 2023 18:33:45 +0100
Subject: [PATCH 615/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFileMetadataCountPerContentType using JPA Criteria

---
 .../DatasetVersionFilesServiceBean.java       | 34 ++++++++-----------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index c2efa72579f..aa92f44c302 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -89,17 +89,17 @@ public long getFileMetadataCount(DatasetVersion datasetVersion, FileSearchCriter
      * @return Map<String, Long> of file metadata counts per content type
      */
     public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<Tuple> baseQuery = queryFactory
-                .select(fileMetadata.dataFile.contentType, fileMetadata.count())
-                .from(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                .groupBy(fileMetadata.dataFile.contentType);
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        List<Tuple> contentTypeOccurrences = baseQuery.fetch();
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        Path<String> contentType = fileMetadataRoot.get("dataFile").get("contentType");
+        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
+        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
+        criteriaQuery.multiselect(contentType, criteriaBuilder.count(contentType)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate)).groupBy(contentType);
+        List<jakarta.persistence.Tuple> contentTypeOccurrences = em.createQuery(criteriaQuery).getResultList();
         Map<String, Long> result = new HashMap<>();
-        for (Tuple occurrence : contentTypeOccurrences) {
-            result.put(occurrence.get(fileMetadata.dataFile.contentType), occurrence.get(fileMetadata.count()));
+        for (jakarta.persistence.Tuple occurrence : contentTypeOccurrences) {
+            result.put(occurrence.get(0, String.class), occurrence.get(1, Long.class));
         }
         return result;
     }
@@ -234,22 +234,16 @@ private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion da
 
     private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus accessStatus, CriteriaBuilder criteriaBuilder, Root<FileMetadata> fileMetadataRoot) {
         Path<Object> dataFile = fileMetadataRoot.get("dataFile");
-
         Path<Object> embargo = dataFile.get("embargo");
         Predicate activelyEmbargoedPredicate = criteriaBuilder.greaterThanOrEqualTo(embargo.<Date>get("dateAvailable"), criteriaBuilder.currentDate());
         Predicate inactivelyEmbargoedPredicate = criteriaBuilder.isNull(embargo);
-
         Path<Boolean> isRestricted = dataFile.get("restricted");
         Predicate isRestrictedPredicate = criteriaBuilder.isTrue(isRestricted);
         Predicate isUnrestrictedPredicate = criteriaBuilder.isFalse(isRestricted);
-
         return switch (accessStatus) {
-            case EmbargoedThenRestricted ->
-                    criteriaBuilder.and(activelyEmbargoedPredicate, isRestrictedPredicate);
-            case EmbargoedThenPublic ->
-                    criteriaBuilder.and(activelyEmbargoedPredicate, isUnrestrictedPredicate);
-            case Restricted ->
-                    criteriaBuilder.and(inactivelyEmbargoedPredicate, isRestrictedPredicate);
+            case EmbargoedThenRestricted -> criteriaBuilder.and(activelyEmbargoedPredicate, isRestrictedPredicate);
+            case EmbargoedThenPublic -> criteriaBuilder.and(activelyEmbargoedPredicate, isUnrestrictedPredicate);
+            case Restricted -> criteriaBuilder.and(inactivelyEmbargoedPredicate, isRestrictedPredicate);
             case Public -> criteriaBuilder.and(inactivelyEmbargoedPredicate, isUnrestrictedPredicate);
         };
     }
@@ -279,7 +273,7 @@ private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAcces
         return accessStatusExpression;
     }
 
-    private Predicate createSearchCriteriaPredicate(FileSearchCriteria searchCriteria, CriteriaBuilder criteriaBuilder, CriteriaQuery<Long> criteriaQuery, Root<FileMetadata> fileMetadataRoot) {
+    private Predicate createSearchCriteriaPredicate(FileSearchCriteria searchCriteria, CriteriaBuilder criteriaBuilder, CriteriaQuery<?> criteriaQuery, Root<FileMetadata> fileMetadataRoot) {
         List<Predicate> predicates = new ArrayList<>();
         String contentType = searchCriteria.getContentType();
         if (contentType != null) {

From 8fbbaf78aa48ccff5c9f10f9078758ac6c06069f Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 24 Oct 2023 18:44:52 +0100
Subject: [PATCH 616/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFileMetadataCountPerCategoryName using JPA Criteria

---
 .../DatasetVersionFilesServiceBean.java       | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index aa92f44c302..622f7883d42 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -92,9 +92,9 @@ public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datas
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
         CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
         Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
-        Path<String> contentType = fileMetadataRoot.get("dataFile").get("contentType");
         Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
         Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
+        Path<String> contentType = fileMetadataRoot.get("dataFile").get("contentType");
         criteriaQuery.multiselect(contentType, criteriaBuilder.count(contentType)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate)).groupBy(contentType);
         List<jakarta.persistence.Tuple> contentTypeOccurrences = em.createQuery(criteriaQuery).getResultList();
         Map<String, Long> result = new HashMap<>();
@@ -111,18 +111,20 @@ public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datas
      * @param searchCriteria for counting only files matching this criteria
      * @return Map<String, Long> of file metadata counts per category name
      */
+    // TODO: Refactor remove duplication with getFileMetadataCountPerContentType
     public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<Tuple> baseQuery = queryFactory
-                .select(dataFileCategory.name, fileMetadata.count())
-                .from(dataFileCategory, fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.fileCategories.contains(dataFileCategory)))
-                .groupBy(dataFileCategory.name);
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        List<Tuple> categoryNameOccurrences = baseQuery.fetch();
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
+        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
+        Root<DataFileCategory> dataFileCategoryRoot = criteriaQuery.from(DataFileCategory.class);
+        Path<String> categoryName = dataFileCategoryRoot.get("name");
+        criteriaQuery.multiselect(categoryName, criteriaBuilder.count(fileMetadataRoot)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate, dataFileCategoryRoot.in(fileMetadataRoot.get("fileCategories")))).groupBy(categoryName);
+        List<jakarta.persistence.Tuple> categoryNameOccurrences = em.createQuery(criteriaQuery).getResultList();
         Map<String, Long> result = new HashMap<>();
-        for (Tuple occurrence : categoryNameOccurrences) {
-            result.put(occurrence.get(dataFileCategory.name), occurrence.get(fileMetadata.count()));
+        for (jakarta.persistence.Tuple occurrence : categoryNameOccurrences) {
+            result.put(occurrence.get(0, String.class), occurrence.get(1, Long.class));
         }
         return result;
     }

From f9a2f2a4038a727b1487b8ddcf385278600ee0af Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Tue, 24 Oct 2023 18:57:13 +0100
Subject: [PATCH 617/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFileMetadataCountPerTabularTagName using JPA Criteria

---
 .../DatasetVersionFilesServiceBean.java       | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 622f7883d42..e57af63b234 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -136,18 +136,20 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
      * @param searchCriteria for counting only files matching this criteria
      * @return Map<DataFileTag.TagType, Long> of file metadata counts per DataFileTag.TagType
      */
+    // TODO: Refactor remove duplication with getFileMetadataCountPerContentType
     public Map<DataFileTag.TagType, Long> getFileMetadataCountPerTabularTagName(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<Tuple> baseQuery = queryFactory
-                .select(dataFileTag.type, fileMetadata.count())
-                .from(dataFileTag, fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)))
-                .groupBy(dataFileTag.type);
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        List<Tuple> tagNameOccurrences = baseQuery.fetch();
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
+        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
+        Root<DataFileTag> dataFileTagRoot = criteriaQuery.from(DataFileTag.class);
+        Path<DataFileTag.TagType> dataFileTagType = dataFileTagRoot.get("type");
+        criteriaQuery.multiselect(dataFileTagType, criteriaBuilder.count(fileMetadataRoot)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate, dataFileTagRoot.in(fileMetadataRoot.get("dataFile").get("dataFileTags")))).groupBy(dataFileTagType);
+        List<jakarta.persistence.Tuple> tagNameOccurrences = em.createQuery(criteriaQuery).getResultList();
         Map<DataFileTag.TagType, Long> result = new HashMap<>();
-        for (Tuple occurrence : tagNameOccurrences) {
-            result.put(occurrence.get(dataFileTag.type), occurrence.get(fileMetadata.count()));
+        for (jakarta.persistence.Tuple occurrence : tagNameOccurrences) {
+            result.put(occurrence.get(0, DataFileTag.TagType.class), occurrence.get(1, Long.class));
         }
         return result;
     }

From 92e59176e5c3ca248e4dd0ee26dd6432173fdaa0 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 24 Oct 2023 15:17:44 -0400
Subject: [PATCH 618/922] adding rel="nofollow" to the links throughout the
 collection page. #9635

---
 src/main/webapp/search-include-fragment.xhtml | 46 +++++++++----------
 src/main/webapp/search-include-pager.xhtml    | 10 ++--
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml
index 8397a14136e..80a0a69b950 100644
--- a/src/main/webapp/search-include-fragment.xhtml
+++ b/src/main/webapp/search-include-fragment.xhtml
@@ -36,7 +36,7 @@
 
                         <p:remoteCommand name="submitsearch" action="#{SearchIncludeFragment.searchRedirect(dataverseRedirectPage, DataversePage.dataverse)}"/>
                     </div>
-                    <h:outputLink id="advsearchlink" value="#{widgetWrapper.wrapURL('/search/advanced.xhtml')}">
+                    <h:outputLink rel="nofollow" id="advsearchlink" value="#{widgetWrapper.wrapURL('/search/advanced.xhtml')}">
                         <f:param name="dataverseIdentifier" value="#{SearchIncludeFragment.dataverse.alias}"/>
                         <h:outputText value="#{bundle['dataverse.search.advancedSearch']}"/>
                         <!--<f:param name="q" value="#{SearchIncludeFragment.query}" disable="#{empty SearchIncludeFragment.query}"/>-->
@@ -108,7 +108,7 @@
             <h:form id="facetType" rendered="#{!SearchIncludeFragment.facetsDisabled}">
                 <!--DATAVERSE TOGGLE-->
                 <div class="clearfix">
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('dataverses') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('dataverses') == null}">
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('dataverses') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('dataverses') == null}">
                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                         <f:param name="types" value="#{SearchIncludeFragment.getNewSelectedTypes('dataverses')}"/>
                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -121,7 +121,7 @@
                         <p:selectBooleanCheckbox value="#{SearchIncludeFragment.selectedTypesList.contains('dataverses') ? true : false}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('dataverses') == null ? true : false}" ariaLabel="#{bundle['facet.collection.label']}"/>
                     </h:outputLink>
                     <!--DATAVERSES ONLY-->
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" disabled="#{false}" styleClass="facetTypeLink chkbox-label-margin #{SearchIncludeFragment.selectedTypesList.contains('dataverses') ? 'facetSelected': ''}">
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" disabled="#{false}" styleClass="facetTypeLink chkbox-label-margin #{SearchIncludeFragment.selectedTypesList.contains('dataverses') ? 'facetSelected': ''}">
                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                         <f:param name="types" value="dataverses"/>
                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -140,7 +140,7 @@
                 </div>
                 <!--DATASETS TOGGLE-->
                 <div class="clearfix">
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('datasets') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('datasets') == null}">
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('datasets') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('datasets') == null}">
                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                         <f:param name="types" value="#{SearchIncludeFragment.getNewSelectedTypes('datasets')}"/>
                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -153,7 +153,7 @@
                         <p:selectBooleanCheckbox value="#{SearchIncludeFragment.selectedTypesList.contains('datasets') ? true : false}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('datasets') == null ? true : false}" ariaLabel="#{bundle['facet.dataset.label']}"/>
                     </h:outputLink>
                     <!--DATASETS ONLY-->
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" disabled="#{false}" styleClass="facetTypeLink chkbox-label-margin #{SearchIncludeFragment.selectedTypesList.contains('datasets') ? 'facetSelected': ''}">
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" disabled="#{false}" styleClass="facetTypeLink chkbox-label-margin #{SearchIncludeFragment.selectedTypesList.contains('datasets') ? 'facetSelected': ''}">
                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                         <f:param name="types" value="datasets"/>
                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -172,7 +172,7 @@
                 </div>
                 <!--FILES TOGGLE-->
                 <div class="clearfix">
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('files') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('files') == null}">
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" styleClass="facetTypeChBox facetLink #{SearchIncludeFragment.selectedTypesList.contains('files') ? 'facetSelected': ''}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('files') == null}">
                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                         <f:param name="types" value="#{SearchIncludeFragment.getNewSelectedTypes('files')}"/>
                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -185,7 +185,7 @@
                         <p:selectBooleanCheckbox value="#{SearchIncludeFragment.selectedTypesList.contains('files') ? true : false}" disabled="#{SearchIncludeFragment.getNewSelectedTypes('files') == null ? true : false}" ariaLabel="#{bundle['facet.datafile.label']}"/>
                     </h:outputLink>
                     <!--FILES ONLY-->
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" disabled="#{false}" styleClass="facetTypeLink chkbox-label-margin #{SearchIncludeFragment.selectedTypesList.contains('files') ? 'facetSelected': ''}">
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" disabled="#{false}" styleClass="facetTypeLink chkbox-label-margin #{SearchIncludeFragment.selectedTypesList.contains('files') ? 'facetSelected': ''}">
                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                         <f:param name="types" value="files"/>
                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -209,7 +209,7 @@
                 <p:dataList id="facetCategoryList" value="#{DataversePage.facetCategoryList}" var="facetCategory">
                     <h:outputText value="#{facetCategory.friendlyName}" styleClass="facetCategoryName"/>
                     <p:dataList value="#{facetCategory.facetLabel}" var="facetLabel" rows="#{DataversePage.getNumberOfFacets(facetCategory.name,5)}">
-                        <h:outputLink value="#{widgetWrapper.wrapURL(page)}" rendered="#{!DataversePage.filterQueries.contains(facetLabel.filterQuery)}" styleClass="facetLink">
+                        <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" rendered="#{!DataversePage.filterQueries.contains(facetLabel.filterQuery)}" styleClass="facetLink">
                             <h:outputText value="#{facetLabel.name}">
                                 <c:if test="#{!cvocConf.isEmpty()}">
                                     <f:passThroughAttribute name="lang" value="#{DatasetPage.getFieldLanguage(cvocConf.get(facetCategory.datasetFieldTypeId).getString('languages'))}" />
@@ -230,7 +230,7 @@
                             <f:param name="order" value="#{DataversePage.sortOrder}"/>
                         </h:outputLink>
 
-                        <h:outputLink value="#{widgetWrapper.wrapURL(page)}" rendered="#{DataversePage.filterQueries.contains(facetLabel.filterQuery)}" styleClass="facetLink facetSelected">
+                        <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" rendered="#{DataversePage.filterQueries.contains(facetLabel.filterQuery)}" styleClass="facetLink facetSelected">
                             <h:outputText value="#{facetLabel.name}">
                                 <c:if test="#{!cvocConf.isEmpty()}">
                                     <f:passThroughAttribute name="lang" value="#{DatasetPage.getFieldLanguage(cvocConf.get(facetCategory.datasetFieldTypeId).getString('languages'))}" />
@@ -308,7 +308,7 @@
                     </ui:fragment>
                     <ui:fragment rendered="#{SearchIncludeFragment.errorFromSolr != null}">
                         <p>
-                            <a data-toggle="collapse" data-parent="#emptyResults" href="#technicalDetails" aria-expanded="true" aria-controls="technicalDetails">[+] #{bundle['dataverse.results.empty.link.technicalDetails']}</a>
+                            <a rel="nofollow" data-toggle="collapse" data-parent="#emptyResults" href="#technicalDetails" aria-expanded="true" aria-controls="technicalDetails">[+] #{bundle['dataverse.results.empty.link.technicalDetails']}</a>
                         </p>
                         <div id="technicalDetails" class="collapse">
                             <pre><h:outputText value="#{SearchIncludeFragment.errorFromSolr}"/></pre>
@@ -379,7 +379,7 @@
                 <ui:repeat value="#{SearchIncludeFragment.filterQueries}" var="filterQuery" rendered="#{!SearchIncludeFragment.wasSolrErrorEncountered()}">
                     <ui:param name="friendlyNames" value="#{SearchIncludeFragment.getFriendlyNamesFromFilterQuery(filterQuery)}"/>
                     <ui:param name="filterType" value="#{SearchIncludeFragment.getTypeFromFilterQuery(filterQuery)}"/>
-                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}" styleClass="facetLink facetSelected" rendered="#{not empty friendlyNames
+                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}" styleClass="facetLink facetSelected" rendered="#{not empty friendlyNames
                                 and filterType != DataversePage.searchFieldType and filterType != DataversePage.searchFieldSubtree}">
                         <h:outputText value="#{friendlyNames.get(0)}: "/>
                         <h:outputText value="#{friendlyNames.get(1)}">
@@ -417,7 +417,7 @@
                         </button>
                         <ul class="dropdown-menu pull-right text-left">
                             <li>
-                                <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                                <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                                     <h:outputText value="#{bundle['dataverse.results.btn.sort.option.nameAZ']}" styleClass="#{SearchIncludeFragment.sortedByNameAsc ? 'highlightBold' : ''}"/>
                                     <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                                     <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -430,7 +430,7 @@
                                 </h:outputLink>
                             </li>
                             <li>
-                                <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                                <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                                     <h:outputText value="#{bundle['dataverse.results.btn.sort.option.nameZA']}" styleClass="#{SearchIncludeFragment.sortedByNameDesc ? 'highlightBold' : ''}"/>
                                     <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                                     <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -443,7 +443,7 @@
                                 </h:outputLink>
                             </li>
                             <li>
-                                <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                                <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                                     <h:outputText value="#{bundle['dataverse.results.btn.sort.option.newest']}" styleClass="#{SearchIncludeFragment.sortedByReleaseDateDesc? 'highlightBold' : ''}"/>
                                     <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                                     <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -456,7 +456,7 @@
                                 </h:outputLink>
                             </li>
                             <li>
-                                <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                                <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                                     <h:outputText value="#{bundle['dataverse.results.btn.sort.option.oldest']}" styleClass="#{SearchIncludeFragment.sortedByReleaseDateAsc? 'highlightBold' : ''}"/>
                                     <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                                     <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -470,7 +470,7 @@
                             </li>
                             <ui:fragment rendered="#{SearchIncludeFragment.mode == SearchIncludeFragment.searchModeString}">
                                 <li>
-                                    <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                                    <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                                         <h:outputText value="#{bundle['dataverse.results.btn.sort.option.relevance']}" styleClass="#{SearchIncludeFragment.sortedByRelevance ? 'highlightBold' : ''}"/>
                                         <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                                         <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -516,7 +516,7 @@
                             <h:outputText value="#{bundle['dataset.versionUI.unpublished']}" styleClass="label label-warning" rendered="#{result.unpublishedState}"/>
                         </div>
                         <div class="card-preview-icon-block text-center">
-                            <h:outputLink value="#{!SearchIncludeFragment.rootDv and !result.isInTree ? dvUrl : widgetWrapper.wrapURL(dvUrl)}" target="#{!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView ? '_blank' : ''}">
+                            <h:outputLink rel="nofollow" value="#{!SearchIncludeFragment.rootDv and !result.isInTree ? dvUrl : widgetWrapper.wrapURL(dvUrl)}" target="#{!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView ? '_blank' : ''}">
                                 <img src="#{result.imageUrl}" jsf:rendered="#{!empty result.imageUrl}" alt="#{of:format1(bundle['alt.logo'], result.name)}"/>
                                 <span class="icon-dataverse text-brand" jsf:rendered="#{empty result.imageUrl}"/>
                                 <f:passThroughAttribute name="aria-label" value="#{of:format1(bundle['alt.logo'], result.name)}"/>
@@ -526,7 +526,7 @@
                         <h:outputText value="#{result.dateToDisplayOnCard}" styleClass="text-muted" style="margin-right:.5em;"/>
 
                         <c:set var="dvParentUrl" value="/dataverse/#{result.dataverseParentAlias}"/>
-                        <h:outputLink value="#{!SearchIncludeFragment.rootDv and !result.isInTree ? dvParentUrl : widgetWrapper.wrapURL(dvParentUrl)}" target="#{!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView ? '_blank' : ''}" rendered="#{result.parent.id != SearchIncludeFragment.dataverse.id}">
+                        <h:outputLink rel="nofollow" value="#{!SearchIncludeFragment.rootDv and !result.isInTree ? dvParentUrl : widgetWrapper.wrapURL(dvParentUrl)}" target="#{!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView ? '_blank' : ''}" rendered="#{result.parent.id != SearchIncludeFragment.dataverse.id}">
                             <h:outputText value="#{result.parent.get('name')}"/>
                         </h:outputLink>
 
@@ -579,7 +579,7 @@
                             <h:outputText value="#{bundle['incomplete']}" styleClass="label label-danger" rendered="#{!SearchIncludeFragment.isValid(result)}"/>
                         </div>
                         <div class="card-preview-icon-block text-center">
-                            <a href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? result.datasetUrl : widgetWrapper.wrapURL(result.datasetUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}" aria-label="#{result.title}">
+                            <a rel="nofollow" href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? result.datasetUrl : widgetWrapper.wrapURL(result.datasetUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}" aria-label="#{result.title}">
                                 <img src="#{result.imageUrl}" jsf:rendered="#{!empty result.imageUrl and !result.deaccessionedState}" alt="#{result.title}"/>
                                 <span class="icon-dataset" jsf:rendered="#{empty result.imageUrl or result.deaccessionedState}"/>
                             </a>
@@ -590,7 +590,7 @@
                         <h:outputText styleClass="text-muted" value=" - " rendered="#{result.parent.id != SearchIncludeFragment.dataverse.id}"/>
                         
                         <c:set var="dvParentAlias" value="/dataverse/#{result.dataverseAlias}"/>
-                        <h:outputLink value="#{!SearchIncludeFragment.rootDv and !result.isInTree ? dvParentAlias : widgetWrapper.wrapURL(dvParentAlias)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}" rendered="#{result.parent.id != SearchIncludeFragment.dataverse.id}">
+                        <h:outputLink rel="nofollow" value="#{!SearchIncludeFragment.rootDv and !result.isInTree ? dvParentAlias : widgetWrapper.wrapURL(dvParentAlias)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}" rendered="#{result.parent.id != SearchIncludeFragment.dataverse.id}">
                             <h:outputText value="#{result.parent.get('name')}"/>
                         </h:outputLink>
 
@@ -642,7 +642,7 @@
 
                             <span class="text-danger glyphicon glyphicon-lock" data-toggle="tooltip" title="#{bundle.restricted}" jsf:rendered="#{result.entity.restricted and !permissionsWrapper.hasDownloadFilePermission(result.entity)}"/>
                             <span class="text-success icon-unlock" data-toggle="tooltip" title="#{bundle.restrictedaccess}" jsf:rendered="#{result.entity.restricted and permissionsWrapper.hasDownloadFilePermission(result.entity)}"/>
-                            <a href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? (result.harvested ? result.fileDatasetUrl : result.fileUrl) : widgetWrapper.wrapURL(result.harvested ? result.fileDatasetUrl : result.fileUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}">
+                            <a rel="nofollow" href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? (result.harvested ? result.fileDatasetUrl : result.fileUrl) : widgetWrapper.wrapURL(result.harvested ? result.fileDatasetUrl : result.fileUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}">
                                 <h:outputText value="#{result.name}" style="padding:4px 0;" rendered="#{result.nameHighlightSnippet == null}"/>
                                 <h:outputText value="#{result.nameHighlightSnippet}" style="padding:4px 0;" rendered="#{result.nameHighlightSnippet != null}" escape="false"/>
                                 <h:outputText value=" (#{result.entityId})" style="padding:4px 0;" rendered="#{dataverseSession.debug}"/></a>
@@ -654,7 +654,7 @@
                         </div>
 
                         <div class="card-preview-icon-block text-center">
-                            <a href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? (result.harvested ? result.fileDatasetUrl : result.fileUrl) : widgetWrapper.wrapURL(result.harvested ? result.fileDatasetUrl : result.fileUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}" aria-label="#{result.name}">
+                            <a rel="nofollow" href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? (result.harvested ? result.fileDatasetUrl : result.fileUrl) : widgetWrapper.wrapURL(result.harvested ? result.fileDatasetUrl : result.fileUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}" aria-label="#{result.name}">
                                 <img src="#{result.imageUrl}" jsf:rendered="#{!empty result.imageUrl}" alt="#{result.name}"/>
                                 <span class="icon-#{dataFileServiceBean.getFileThumbnailClass(result.entity)} text-muted h1"
                                       jsf:rendered="#{empty result.imageUrl}"/>
@@ -663,7 +663,7 @@
 
                         <div class="text-muted">
                             <h:outputText value="#{result.dateToDisplayOnCard} - "/>
-                            <a href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? result.fileDatasetUrl : widgetWrapper.wrapURL(result.fileDatasetUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}">
+                            <a rel="nofollow" href="#{!SearchIncludeFragment.rootDv and !result.isInTree ? result.fileDatasetUrl : widgetWrapper.wrapURL(result.fileDatasetUrl)}" target="#{(!SearchIncludeFragment.rootDv and !result.isInTree and widgetWrapper.widgetView) or result.harvested ? '_blank' : ''}">
                                 <h:outputText value="#{result.parent.get('name')}"/></a>
                         </div>
 
diff --git a/src/main/webapp/search-include-pager.xhtml b/src/main/webapp/search-include-pager.xhtml
index ad8fb316dda..88714099c00 100644
--- a/src/main/webapp/search-include-pager.xhtml
+++ b/src/main/webapp/search-include-pager.xhtml
@@ -15,7 +15,7 @@
                 <ul class="pagination">
                     <!--should take you to page 1-->
                     <li class="#{SearchIncludeFragment.page == '1' ? 'disabled' : ''}">
-                        <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                        <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                             <h:outputText value="&#171;"/>
                             <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                             <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -29,7 +29,7 @@
                         </h:outputLink>
                     </li>
                     <li class="#{SearchIncludeFragment.page == '1' ? 'disabled' : ''}">
-                        <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                        <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                             <h:outputText value="&lt; #{bundle.previous}"/>
                             <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                             <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -46,7 +46,7 @@
                                end="#{Math:min(SearchIncludeFragment.totalPages,SearchIncludeFragment.page+Math:max(2,5-SearchIncludeFragment.page))}"
                                varStatus="pageStatus">
                         <li class="#{SearchIncludeFragment.page == pageStatus.index ? 'active' : ''}">
-                            <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                            <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                                 <h:outputText value="#{pageStatus.index}">
                                     <f:convertNumber pattern="###,###" />
                                 </h:outputText>
@@ -65,7 +65,7 @@
                         </li>
                     </c:forEach>
                     <li class="#{SearchIncludeFragment.page == SearchIncludeFragment.totalPages ? 'disabled' : ''}">
-                        <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                        <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                             <h:outputText value="#{bundle.next} &gt;"/>
                             <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                             <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">
@@ -79,7 +79,7 @@
                         </h:outputLink>
                     </li>
                     <li class="#{SearchIncludeFragment.page == SearchIncludeFragment.totalPages ? 'disabled' : ''}">
-                        <h:outputLink value="#{widgetWrapper.wrapURL(page)}">
+                        <h:outputLink rel="nofollow" value="#{widgetWrapper.wrapURL(page)}">
                             <h:outputText value="&#187;"/>
                             <f:param name="q" value="#{SearchIncludeFragment.query}"/>
                             <c:forEach items="#{SearchIncludeFragment.filterQueries}" var="clickedFilterQuery" varStatus="status">

From 09472413bf23c3ba97eacb79114dd67ba84d19af Mon Sep 17 00:00:00 2001
From: Don Sizemore <don.sizemore@github.com>
Date: Tue, 24 Oct 2023 15:37:13 -0400
Subject: [PATCH 619/922] #10038 document preserving access logging and other
 configuration in 6.0 upgrade steps

---
 doc/release-notes/6.0-release-notes.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/doc/release-notes/6.0-release-notes.md b/doc/release-notes/6.0-release-notes.md
index df916216f5b..858cd604bda 100644
--- a/doc/release-notes/6.0-release-notes.md
+++ b/doc/release-notes/6.0-release-notes.md
@@ -166,6 +166,22 @@ If you are running Payara as a non-root user (and you should be!), **remember no
 
    Your `:MDCLogPath` database setting might be pointing to a Payara 5 directory such as `/usr/local/payara5/glassfish/domains/domain1/logs`. If so, edit this to be Payara 6. You'll probably want to copy your logs over as well.
 
+1. If you've enabled access logging or any other site-specific configuration, be sure to preserve them. For instance, the default domain.xml includes
+
+   ```
+        <http-service>
+        <access-log></access-log>
+   ```
+
+   but you may wish to include
+
+   ```
+        <http-service access-logging-enabled="true">
+        <access-log format="%client.name% %datetime% %request% %status% %response.length% %header.user-agent% %header.referer% %cookie.JSESSIONID% %header.x-forwarded-for%"></access-log>
+   ```
+
+   Be sure to keep a previous copy of your domain.xml for reference.
+
 1. Update systemd unit file (or other init system) from `/usr/local/payara5` to `/usr/local/payara6`, if applicable.
 
 1. Start Payara.

From 7d8135e24fd94b960de8c70025c11074cdfdefdb Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 25 Oct 2023 09:57:17 +0100
Subject: [PATCH 620/922] Refactor: rewritten JPA Criteria queries refactored
 to avoid duplication and improve legibility

---
 .../DatasetVersionFilesServiceBean.java       | 78 +++++++++++--------
 1 file changed, 46 insertions(+), 32 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index e57af63b234..134f873aa4b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -7,7 +7,6 @@
 import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
 
-import com.querydsl.core.Tuple;
 import com.querydsl.core.types.dsl.BooleanExpression;
 import com.querydsl.core.types.dsl.CaseBuilder;
 import com.querydsl.core.types.dsl.DateExpression;
@@ -29,6 +28,7 @@
 import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
 
 import edu.harvard.iq.dataverse.FileSearchCriteria.FileAccessStatus;
+import jakarta.persistence.Tuple;
 import jakarta.persistence.criteria.*;
 
 @Stateless
@@ -75,9 +75,11 @@ public long getFileMetadataCount(DatasetVersion datasetVersion, FileSearchCriter
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
         CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
         Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
-        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
-        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
-        criteriaQuery.select(criteriaBuilder.count(fileMetadataRoot)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate));
+        criteriaQuery
+                .select(criteriaBuilder.count(fileMetadataRoot))
+                .where(criteriaBuilder.and(
+                        createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
+                        createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)));
         return em.createQuery(criteriaQuery).getSingleResult();
     }
 
@@ -90,18 +92,16 @@ public long getFileMetadataCount(DatasetVersion datasetVersion, FileSearchCriter
      */
     public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
-        CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
+        CriteriaQuery<Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
         Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
-        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
-        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
         Path<String> contentType = fileMetadataRoot.get("dataFile").get("contentType");
-        criteriaQuery.multiselect(contentType, criteriaBuilder.count(contentType)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate)).groupBy(contentType);
-        List<jakarta.persistence.Tuple> contentTypeOccurrences = em.createQuery(criteriaQuery).getResultList();
-        Map<String, Long> result = new HashMap<>();
-        for (jakarta.persistence.Tuple occurrence : contentTypeOccurrences) {
-            result.put(occurrence.get(0, String.class), occurrence.get(1, Long.class));
-        }
-        return result;
+        criteriaQuery
+                .multiselect(contentType, criteriaBuilder.count(contentType))
+                .where(criteriaBuilder.and(
+                        createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
+                        createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)))
+                .groupBy(contentType);
+        return getStringLongMapResultFromQuery(criteriaQuery);
     }
 
     /**
@@ -111,22 +111,20 @@ public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datas
      * @param searchCriteria for counting only files matching this criteria
      * @return Map<String, Long> of file metadata counts per category name
      */
-    // TODO: Refactor remove duplication with getFileMetadataCountPerContentType
     public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
-        CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
+        CriteriaQuery<Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
         Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
-        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
-        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
         Root<DataFileCategory> dataFileCategoryRoot = criteriaQuery.from(DataFileCategory.class);
         Path<String> categoryName = dataFileCategoryRoot.get("name");
-        criteriaQuery.multiselect(categoryName, criteriaBuilder.count(fileMetadataRoot)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate, dataFileCategoryRoot.in(fileMetadataRoot.get("fileCategories")))).groupBy(categoryName);
-        List<jakarta.persistence.Tuple> categoryNameOccurrences = em.createQuery(criteriaQuery).getResultList();
-        Map<String, Long> result = new HashMap<>();
-        for (jakarta.persistence.Tuple occurrence : categoryNameOccurrences) {
-            result.put(occurrence.get(0, String.class), occurrence.get(1, Long.class));
-        }
-        return result;
+        criteriaQuery
+                .multiselect(categoryName, criteriaBuilder.count(fileMetadataRoot))
+                .where(criteriaBuilder.and(
+                                createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
+                                createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)),
+                        dataFileCategoryRoot.in(fileMetadataRoot.get("fileCategories")))
+                .groupBy(categoryName);
+        return getStringLongMapResultFromQuery(criteriaQuery);
     }
 
     /**
@@ -136,19 +134,22 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
      * @param searchCriteria for counting only files matching this criteria
      * @return Map<DataFileTag.TagType, Long> of file metadata counts per DataFileTag.TagType
      */
-    // TODO: Refactor remove duplication with getFileMetadataCountPerContentType
     public Map<DataFileTag.TagType, Long> getFileMetadataCountPerTabularTagName(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
         CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
-        CriteriaQuery<jakarta.persistence.Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
+        CriteriaQuery<Tuple> criteriaQuery = criteriaBuilder.createTupleQuery();
         Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
-        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
-        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
         Root<DataFileTag> dataFileTagRoot = criteriaQuery.from(DataFileTag.class);
         Path<DataFileTag.TagType> dataFileTagType = dataFileTagRoot.get("type");
-        criteriaQuery.multiselect(dataFileTagType, criteriaBuilder.count(fileMetadataRoot)).where(criteriaBuilder.and(basePredicate, searchCriteriaPredicate, dataFileTagRoot.in(fileMetadataRoot.get("dataFile").get("dataFileTags")))).groupBy(dataFileTagType);
-        List<jakarta.persistence.Tuple> tagNameOccurrences = em.createQuery(criteriaQuery).getResultList();
+        criteriaQuery
+                .multiselect(dataFileTagType, criteriaBuilder.count(fileMetadataRoot))
+                .where(criteriaBuilder.and(
+                        createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
+                        createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot),
+                        dataFileTagRoot.in(fileMetadataRoot.get("dataFile").get("dataFileTags"))))
+                .groupBy(dataFileTagType);
+        List<Tuple> tagNameOccurrences = em.createQuery(criteriaQuery).getResultList();
         Map<DataFileTag.TagType, Long> result = new HashMap<>();
-        for (jakarta.persistence.Tuple occurrence : tagNameOccurrences) {
+        for (Tuple occurrence : tagNameOccurrences) {
             result.put(occurrence.get(0, DataFileTag.TagType.class), occurrence.get(1, Long.class));
         }
         return result;
@@ -382,4 +383,17 @@ private long getArchivalFilesSize(DatasetVersion datasetVersion, boolean ignoreT
         }
         return (result == null) ? 0 : result;
     }
+
+    private Predicate createFileMetadataFromDatasetVersionPredicate(DatasetVersion datasetVersion, CriteriaBuilder criteriaBuilder, Root<FileMetadata> fileMetadataRoot) {
+        return criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
+    }
+
+    private Map<String, Long> getStringLongMapResultFromQuery(CriteriaQuery<Tuple> criteriaQuery) {
+        List<Tuple> categoryNameOccurrences = em.createQuery(criteriaQuery).getResultList();
+        Map<String, Long> result = new HashMap<>();
+        for (Tuple occurrence : categoryNameOccurrences) {
+            result.put(occurrence.get(0, String.class), occurrence.get(1, Long.class));
+        }
+        return result;
+    }
 }

From 2ffec93586f1afd4fcc51314b05cdef7d4301b83 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 25 Oct 2023 10:28:23 +0100
Subject: [PATCH 621/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFileMetadataCountByAccessStatus using JPA Criteria

---
 .../DatasetVersionFilesServiceBean.java       | 43 +++++++++----------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 134f873aa4b..e8dde2fd73a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -77,9 +77,7 @@ public long getFileMetadataCount(DatasetVersion datasetVersion, FileSearchCriter
         Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
         criteriaQuery
                 .select(criteriaBuilder.count(fileMetadataRoot))
-                .where(criteriaBuilder.and(
-                        createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
-                        createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)));
+                .where(createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot));
         return em.createQuery(criteriaQuery).getSingleResult();
     }
 
@@ -97,9 +95,7 @@ public Map<String, Long> getFileMetadataCountPerContentType(DatasetVersion datas
         Path<String> contentType = fileMetadataRoot.get("dataFile").get("contentType");
         criteriaQuery
                 .multiselect(contentType, criteriaBuilder.count(contentType))
-                .where(criteriaBuilder.and(
-                        createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
-                        createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)))
+                .where(createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot))
                 .groupBy(contentType);
         return getStringLongMapResultFromQuery(criteriaQuery);
     }
@@ -120,9 +116,8 @@ public Map<String, Long> getFileMetadataCountPerCategoryName(DatasetVersion data
         criteriaQuery
                 .multiselect(categoryName, criteriaBuilder.count(fileMetadataRoot))
                 .where(criteriaBuilder.and(
-                                createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
-                                createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)),
-                        dataFileCategoryRoot.in(fileMetadataRoot.get("fileCategories")))
+                        createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot),
+                        dataFileCategoryRoot.in(fileMetadataRoot.get("fileCategories"))))
                 .groupBy(categoryName);
         return getStringLongMapResultFromQuery(criteriaQuery);
     }
@@ -143,8 +138,7 @@ public Map<DataFileTag.TagType, Long> getFileMetadataCountPerTabularTagName(Data
         criteriaQuery
                 .multiselect(dataFileTagType, criteriaBuilder.count(fileMetadataRoot))
                 .where(criteriaBuilder.and(
-                        createFileMetadataFromDatasetVersionPredicate(datasetVersion, criteriaBuilder, fileMetadataRoot),
-                        createSearchCriteriaPredicate(searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot),
+                        createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot),
                         dataFileTagRoot.in(fileMetadataRoot.get("dataFile").get("dataFileTags"))))
                 .groupBy(dataFileTagType);
         List<Tuple> tagNameOccurrences = em.createQuery(criteriaQuery).getResultList();
@@ -220,12 +214,15 @@ private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map<File
     }
 
     private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, FileAccessStatus accessStatus, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<FileMetadata> baseQuery = queryFactory
-                .selectFrom(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus)));
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        return baseQuery.stream().count();
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        criteriaQuery
+                .select(criteriaBuilder.count(fileMetadataRoot))
+                .where(criteriaBuilder.and(
+                        createSearchCriteriaAccessStatusPredicate(accessStatus, criteriaBuilder, fileMetadataRoot),
+                        createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)));
+        return em.createQuery(criteriaQuery).getSingleResult();
     }
 
     private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileOrderCriteria orderCriteria) {
@@ -278,8 +275,14 @@ private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAcces
         return accessStatusExpression;
     }
 
-    private Predicate createSearchCriteriaPredicate(FileSearchCriteria searchCriteria, CriteriaBuilder criteriaBuilder, CriteriaQuery<?> criteriaQuery, Root<FileMetadata> fileMetadataRoot) {
+    private Predicate createSearchCriteriaPredicate(DatasetVersion datasetVersion,
+                                                    FileSearchCriteria searchCriteria,
+                                                    CriteriaBuilder criteriaBuilder,
+                                                    CriteriaQuery<?> criteriaQuery,
+                                                    Root<FileMetadata> fileMetadataRoot) {
         List<Predicate> predicates = new ArrayList<>();
+        Predicate basePredicate = criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
+        predicates.add(basePredicate);
         String contentType = searchCriteria.getContentType();
         if (contentType != null) {
             predicates.add(criteriaBuilder.equal(fileMetadataRoot.get("dataFile").<String>get("contentType"), contentType));
@@ -384,10 +387,6 @@ private long getArchivalFilesSize(DatasetVersion datasetVersion, boolean ignoreT
         return (result == null) ? 0 : result;
     }
 
-    private Predicate createFileMetadataFromDatasetVersionPredicate(DatasetVersion datasetVersion, CriteriaBuilder criteriaBuilder, Root<FileMetadata> fileMetadataRoot) {
-        return criteriaBuilder.equal(fileMetadataRoot.get("datasetVersion").<String>get("id"), datasetVersion.getId());
-    }
-
     private Map<String, Long> getStringLongMapResultFromQuery(CriteriaQuery<Tuple> criteriaQuery) {
         List<Tuple> categoryNameOccurrences = em.createQuery(criteriaQuery).getResultList();
         Map<String, Long> result = new HashMap<>();

From a760807293537c2a0868ea928416b9755ce2485e Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 25 Oct 2023 10:54:56 +0100
Subject: [PATCH 622/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFilesDownloadSize using JPA Criteria

---
 .../DatasetVersionFilesServiceBean.java       | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index e8dde2fd73a..661af5c210e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -2,7 +2,6 @@
 
 import edu.harvard.iq.dataverse.QDataFileCategory;
 import edu.harvard.iq.dataverse.QDataFileTag;
-import edu.harvard.iq.dataverse.QDataTable;
 import edu.harvard.iq.dataverse.QDvObject;
 import edu.harvard.iq.dataverse.QEmbargo;
 import edu.harvard.iq.dataverse.QFileMetadata;
@@ -42,7 +41,6 @@ public class DatasetVersionFilesServiceBean implements Serializable {
     private final QDvObject dvObject = QDvObject.dvObject;
     private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
     private final QDataFileTag dataFileTag = QDataFileTag.dataFileTag;
-    private final QDataTable dataTable = QDataTable.dataTable;
 
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
@@ -361,29 +359,34 @@ private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> qu
     }
 
     private long getOriginalTabularFilesSize(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<?> baseQuery = queryFactory
-                .from(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()))
-                .from(dataTable)
-                .where(dataTable.dataFile.eq(fileMetadata.dataFile));
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        Long result = baseQuery.select(dataTable.originalFileSize.sum()).fetchFirst();
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        Root<DataTable> dataTableRoot = criteriaQuery.from(DataTable.class);
+        criteriaQuery
+                .select(criteriaBuilder.sum(dataTableRoot.get("originalFileSize")))
+                .where(criteriaBuilder.and(
+                        criteriaBuilder.equal(dataTableRoot.get("dataFile"), fileMetadataRoot.get("dataFile")),
+                        createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot)));
+        Long result = em.createQuery(criteriaQuery).getSingleResult();
         return (result == null) ? 0 : result;
     }
 
     private long getArchivalFilesSize(DatasetVersion datasetVersion, boolean ignoreTabular, FileSearchCriteria searchCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<?> baseQuery = queryFactory
-                .from(fileMetadata)
-                .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        Long result;
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<Long> criteriaQuery = criteriaBuilder.createQuery(Long.class);
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        Predicate searchCriteriaPredicate = createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot);
+        Predicate wherePredicate;
         if (ignoreTabular) {
-            result = baseQuery.where(fileMetadata.dataFile.dataTables.isEmpty()).select(fileMetadata.dataFile.filesize.sum()).fetchFirst();
+            wherePredicate = criteriaBuilder.and(searchCriteriaPredicate, criteriaBuilder.isEmpty(fileMetadataRoot.get("dataFile").get("dataTables")));
         } else {
-            result = baseQuery.select(fileMetadata.dataFile.filesize.sum()).fetchFirst();
+            wherePredicate = searchCriteriaPredicate;
         }
+        criteriaQuery
+                .select(criteriaBuilder.sum(fileMetadataRoot.get("dataFile").get("filesize")))
+                .where(wherePredicate);
+        Long result = em.createQuery(criteriaQuery).getSingleResult();
         return (result == null) ? 0 : result;
     }
 

From 653d390f3bf28354174fd12e04aed0fa009548f8 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 25 Oct 2023 13:34:09 +0100
Subject: [PATCH 623/922] Changed: rewritten DatasetVersionFilesServiceBean
 getFileMetadatas using JPA Criteria and removed QueryDSL references from the
 class

---
 .../DatasetVersionFilesServiceBean.java       | 142 ++++--------------
 1 file changed, 31 insertions(+), 111 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 661af5c210e..78fd896c897 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -1,35 +1,20 @@
 package edu.harvard.iq.dataverse;
 
-import edu.harvard.iq.dataverse.QDataFileCategory;
-import edu.harvard.iq.dataverse.QDataFileTag;
-import edu.harvard.iq.dataverse.QDvObject;
-import edu.harvard.iq.dataverse.QEmbargo;
-import edu.harvard.iq.dataverse.QFileMetadata;
-
-import com.querydsl.core.types.dsl.BooleanExpression;
-import com.querydsl.core.types.dsl.CaseBuilder;
-import com.querydsl.core.types.dsl.DateExpression;
-import com.querydsl.core.types.dsl.DateTimeExpression;
-
-import com.querydsl.jpa.impl.JPAQuery;
-import com.querydsl.jpa.impl.JPAQueryFactory;
-
+import edu.harvard.iq.dataverse.FileSearchCriteria.FileAccessStatus;
 import jakarta.ejb.Stateless;
 import jakarta.inject.Named;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
+import jakarta.persistence.Tuple;
+import jakarta.persistence.TypedQuery;
+import jakarta.persistence.criteria.*;
 
 import java.io.Serializable;
 import java.sql.Timestamp;
-import java.time.LocalDate;
 import java.util.*;
 
 import static edu.harvard.iq.dataverse.DataFileTag.TagLabelToTypes;
 
-import edu.harvard.iq.dataverse.FileSearchCriteria.FileAccessStatus;
-import jakarta.persistence.Tuple;
-import jakarta.persistence.criteria.*;
-
 @Stateless
 @Named
 public class DatasetVersionFilesServiceBean implements Serializable {
@@ -37,11 +22,6 @@ public class DatasetVersionFilesServiceBean implements Serializable {
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
 
-    private final QFileMetadata fileMetadata = QFileMetadata.fileMetadata;
-    private final QDvObject dvObject = QDvObject.dvObject;
-    private final QDataFileCategory dataFileCategory = QDataFileCategory.dataFileCategory;
-    private final QDataFileTag dataFileTag = QDataFileTag.dataFileTag;
-
     /**
      * Different criteria to sort the results of FileMetadata queries used in {@link DatasetVersionFilesServiceBean#getFileMetadatas}
      */
@@ -174,16 +154,21 @@ public Map<FileAccessStatus, Long> getFileMetadataCountPerAccessStatus(DatasetVe
      * @return a FileMetadata list from the specified DatasetVersion
      */
     public List<FileMetadata> getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, FileSearchCriteria searchCriteria, FileOrderCriteria orderCriteria) {
-        JPAQuery<FileMetadata> baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria);
-        applyFileSearchCriteriaToQuery(baseQuery, searchCriteria);
-        applyOrderCriteriaToGetFileMetadatasQuery(baseQuery, orderCriteria);
+        CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder();
+        CriteriaQuery<FileMetadata> criteriaQuery = criteriaBuilder.createQuery(FileMetadata.class);
+        Root<FileMetadata> fileMetadataRoot = criteriaQuery.from(FileMetadata.class);
+        criteriaQuery
+                .select(fileMetadataRoot)
+                .where(createSearchCriteriaPredicate(datasetVersion, searchCriteria, criteriaBuilder, criteriaQuery, fileMetadataRoot))
+                .orderBy(createGetFileMetadatasOrder(criteriaBuilder, orderCriteria, fileMetadataRoot));
+        TypedQuery<FileMetadata> typedQuery = em.createQuery(criteriaQuery);
         if (limit != null) {
-            baseQuery.limit(limit);
+            typedQuery.setMaxResults(limit);
         }
         if (offset != null) {
-            baseQuery.offset(offset);
+            typedQuery.setFirstResult(offset);
         }
-        return baseQuery.fetch();
+        return typedQuery.getResultList();
     }
 
     /**
@@ -223,15 +208,6 @@ private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, F
         return em.createQuery(criteriaQuery).getSingleResult();
     }
 
-    private JPAQuery<FileMetadata> createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileOrderCriteria orderCriteria) {
-        JPAQueryFactory queryFactory = new JPAQueryFactory(em);
-        JPAQuery<FileMetadata> baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()));
-        if (orderCriteria == FileOrderCriteria.Newest || orderCriteria == FileOrderCriteria.Oldest) {
-            baseQuery.from(dvObject).where(dvObject.id.eq(fileMetadata.dataFile.id));
-        }
-        return baseQuery;
-    }
-
     private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus accessStatus, CriteriaBuilder criteriaBuilder, Root<FileMetadata> fileMetadataRoot) {
         Path<Object> dataFile = fileMetadataRoot.get("dataFile");
         Path<Object> embargo = dataFile.get("embargo");
@@ -248,31 +224,6 @@ private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus acc
         };
     }
 
-    @Deprecated
-    private BooleanExpression createGetFileMetadatasAccessStatusExpression(FileAccessStatus accessStatus) {
-        QEmbargo embargo = fileMetadata.dataFile.embargo;
-        BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class));
-        BooleanExpression inactivelyEmbargoedExpression = embargo.isNull();
-        BooleanExpression accessStatusExpression;
-        switch (accessStatus) {
-            case EmbargoedThenRestricted:
-                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
-                break;
-            case EmbargoedThenPublic:
-                accessStatusExpression = activelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
-                break;
-            case Restricted:
-                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isTrue());
-                break;
-            case Public:
-                accessStatusExpression = inactivelyEmbargoedExpression.and(fileMetadata.dataFile.restricted.isFalse());
-                break;
-            default:
-                throw new IllegalStateException("Unexpected value: " + accessStatus);
-        }
-        return accessStatusExpression;
-    }
-
     private Predicate createSearchCriteriaPredicate(DatasetVersion datasetVersion,
                                                     FileSearchCriteria searchCriteria,
                                                     CriteriaBuilder criteriaBuilder,
@@ -309,53 +260,22 @@ private Predicate createSearchCriteriaPredicate(DatasetVersion datasetVersion,
         return criteriaBuilder.and(predicates.toArray(new Predicate[]{}));
     }
 
-    @Deprecated
-    private void applyFileSearchCriteriaToQuery(JPAQuery<?> baseQuery, FileSearchCriteria searchCriteria) {
-        String contentType = searchCriteria.getContentType();
-        if (contentType != null) {
-            baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType));
-        }
-        FileAccessStatus accessStatus = searchCriteria.getAccessStatus();
-        if (accessStatus != null) {
-            baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus));
-        }
-        String categoryName = searchCriteria.getCategoryName();
-        if (categoryName != null) {
-            baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory)));
-        }
-        String tabularTagName = searchCriteria.getTabularTagName();
-        if (tabularTagName != null) {
-            baseQuery.from(dataFileTag).where(dataFileTag.type.eq(TagLabelToTypes.get(tabularTagName)).and(fileMetadata.dataFile.dataFileTags.contains(dataFileTag)));
-        }
-        String searchText = searchCriteria.getSearchText();
-        if (searchText != null && !searchText.isEmpty()) {
-            searchText = searchText.trim().toLowerCase();
-            baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText)));
-        }
-    }
-
-    private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery<FileMetadata> query, FileOrderCriteria orderCriteria) {
-        DateTimeExpression<Timestamp> orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate);
-        switch (orderCriteria) {
-            case NameZA:
-                query.orderBy(fileMetadata.label.desc());
-                break;
-            case Newest:
-                query.orderBy(orderByLifetimeExpression.desc());
-                break;
-            case Oldest:
-                query.orderBy(orderByLifetimeExpression.asc());
-                break;
-            case Size:
-                query.orderBy(fileMetadata.dataFile.filesize.asc());
-                break;
-            case Type:
-                query.orderBy(fileMetadata.dataFile.contentType.asc(), fileMetadata.label.asc());
-                break;
-            default:
-                query.orderBy(fileMetadata.label.asc());
-                break;
-        }
+    private Order createGetFileMetadatasOrder(CriteriaBuilder criteriaBuilder,
+                                              FileOrderCriteria orderCriteria,
+                                              Root<FileMetadata> fileMetadataRoot) {
+        Path<Object> label = fileMetadataRoot.get("label");
+        Path<Object> dataFile = fileMetadataRoot.get("dataFile");
+        Path<Timestamp> publicationDate = dataFile.get("publicationDate");
+        Path<Timestamp> createDate = dataFile.get("createDate");
+        Expression<Object> orderByLifetimeExpression = criteriaBuilder.selectCase().when(publicationDate.isNotNull(), publicationDate).otherwise(createDate);
+        return switch (orderCriteria) {
+            case NameZA -> criteriaBuilder.desc(label);
+            case Newest -> criteriaBuilder.desc(orderByLifetimeExpression);
+            case Oldest -> criteriaBuilder.asc(orderByLifetimeExpression);
+            case Size -> criteriaBuilder.asc(dataFile.get("filesize"));
+            case Type -> criteriaBuilder.asc(dataFile.get("contentType"));
+            default -> criteriaBuilder.asc(label);
+        };
     }
 
     private long getOriginalTabularFilesSize(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {

From 99f32fc370d615568bcd45017fdfd1a5b2ec1dcc Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 25 Oct 2023 14:51:10 +0100
Subject: [PATCH 624/922] Removed: QueryDSL library from the application
 dependencies

---
 modules/dataverse-parent/pom.xml |  3 ---
 pom.xml                          | 14 --------------
 2 files changed, 17 deletions(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index 1d99c1cd3d8..db0fa46a952 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -200,9 +200,6 @@
         
         <!-- Container related -->
         <fabric8-dmp.version>0.43.4</fabric8-dmp.version>
-
-        <!-- Persistence -->
-        <querydsl.version>5.0.0</querydsl.version>
     </properties>
     
     <pluginRepositories>
diff --git a/pom.xml b/pom.xml
index bcca9884d50..5536bcccb05 100644
--- a/pom.xml
+++ b/pom.xml
@@ -252,20 +252,6 @@
             <artifactId>expressly</artifactId>
             <scope>provided</scope>
         </dependency>
-
-        <dependency>
-            <groupId>com.querydsl</groupId>
-            <artifactId>querydsl-apt</artifactId>
-            <version>${querydsl.version}</version>
-            <classifier>jakarta</classifier>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.querydsl</groupId>
-            <artifactId>querydsl-jpa</artifactId>
-            <version>${querydsl.version}</version>
-            <classifier>jakarta</classifier>
-        </dependency>
         
         <dependency>
             <groupId>commons-io</groupId>

From 643c55594f4dc3ccd654cc0438d688684c6e5f02 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 25 Oct 2023 10:50:45 -0400
Subject: [PATCH 625/922] an off switch for the facets on the collection page.
 (#9635)

---
 .../search/SearchIncludeFragment.java         | 23 +++++++++++--------
 .../settings/SettingsServiceBean.java         |  7 +++++-
 src/main/java/propertyFiles/Bundle.properties |  3 +++
 src/main/webapp/search-include-fragment.xhtml |  9 ++++----
 4 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 14274a09399..6ba5817bcdd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -22,6 +22,7 @@
 import edu.harvard.iq.dataverse.ThumbnailServiceWrapper;
 import edu.harvard.iq.dataverse.WidgetWrapper;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import java.time.LocalDate;
 import java.util.ArrayList;
@@ -132,7 +133,6 @@ public class SearchIncludeFragment implements java.io.Serializable {
     Map<String, String> staticSolrFieldFriendlyNamesBySolrField = new HashMap<>();
     private boolean solrIsDown = false;
     private boolean solrIsTemporarilyUnavailable = false; 
-    private boolean solrFacetsDisabled = false;
     private Map<String, Integer> numberOfFacets = new HashMap<>();
 //    private boolean showUnpublished;
     List<String> filterQueriesDebug = new ArrayList<>();
@@ -358,7 +358,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             DataverseRequest dataverseRequest = new DataverseRequest(session.getUser(), httpServletRequest);
             List<Dataverse> dataverses = new ArrayList<>();
             dataverses.add(dataverse);
-            solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null);
+            solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null, !isFacetsDisabled(), true);
             if (solrQueryResponse.hasError()){
                 logger.info(solrQueryResponse.getError());
                 setSolrErrorEncountered(true);
@@ -395,7 +395,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 }
             }
             
-            if (selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable()) {
+            if (selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable() && !isFacetsDisabled()) {
                 // If some types are NOT currently selected, we will need to 
                 // run another query to obtain the numbers of the unselected types:
                 
@@ -1096,15 +1096,20 @@ public void setSolrTemporarilyUnavailable(boolean solrIsTemporarilyUnavailable)
         this.solrIsTemporarilyUnavailable = solrIsTemporarilyUnavailable; 
     }
 
+    /**
+     * Indicates that the fragment should not be requesting facets in Solr 
+     * searches and rendering them on the page.
+     * @return true if disabled; false by default 
+     */
     public boolean isFacetsDisabled() {
-        return solrFacetsDisabled;
-    }
-    
-    public void setFacetsDisabled(boolean solrFacetsDisabled) {
-        this.solrFacetsDisabled = solrFacetsDisabled; 
+        // The method is used in rendered="..." logic. So we are using 
+        // SettingsWrapper to make sure we are not looking it up repeatedly 
+        // (settings are not expensive to look up, but 
+        // still).
+        
+        return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacets, false);
     }
     
-    
     public boolean isRootDv() {
         return rootDv;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
index 0aa403a5116..fdd286bd5dc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
@@ -608,7 +608,12 @@ Whether Harvesting (OAI) service is enabled
         /*
          * True/false(default) option deciding whether file PIDs can be enabled per collection - using the Dataverse/collection set attribute API call.
          */
-        AllowEnablingFilePIDsPerCollection
+        AllowEnablingFilePIDsPerCollection,
+        /**
+         * Allows an instance admin to disable Solr search facets on the collection
+         * and dataset pages instantly
+         */
+        DisableSolrFacets
         ;
 
         @Override
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 79887f7e76c..af920a66191 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1006,6 +1006,9 @@ dataverse.results.btn.sort.option.relevance=Relevance
 dataverse.results.cards.foundInMetadata=Found in Metadata Fields:
 dataverse.results.cards.files.tabularData=Tabular Data
 dataverse.results.solrIsDown=Please note: Due to an internal error, browsing and searching is not available.
+dataverse.results.solrIsTemporarilyUnavailable=Search Engine service (Solr) is temporarily unavailable because of high load. Please try again later.
+dataverse.results.solrIsTemporarilyUnavailable.extraText=Note that all the datasets that are part of this collection are accessible via direct links and registered DOIs.
+dataverse.results.solrFacetsDisabled=Facets temporarily unavailable.
 dataverse.theme.title=Theme
 dataverse.theme.inheritCustomization.title=For this dataverse, use the same theme as the parent dataverse.
 dataverse.theme.inheritCustomization.label=Inherit Theme
diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml
index 80a0a69b950..af568170157 100644
--- a/src/main/webapp/search-include-fragment.xhtml
+++ b/src/main/webapp/search-include-fragment.xhtml
@@ -94,10 +94,9 @@
     <div class="row" jsf:rendered="#{SearchIncludeFragment.solrTemporarilyUnavailable}">
         <div class="col-sm-12">
             <div class="bg-danger padding-12 margin-bottom">
-                <!-- @todo: move this text into the bundle -->
-                <h:outputText value="Search Engine service (Solr) is temporarily unavailable because of load issues. Please try again later. " styleClass="highlightBold"/>
+                <h:outputText value="#{bundle['dataverse.results.solrIsTemporarilyUnavailable']}" styleClass="highlightBold"/>
             </div>
-            <pre><h:outputText value="Note that all the datasets that are part of this collection are accessible via direct links and registered DOIs"/></pre>
+            <pre><h:outputText value="#{bundle['dataverse.results.solrIsTemporarilyUnavailable.extraText']}"/></pre>
         </div>
     </div>
 
@@ -263,10 +262,10 @@
                     </ui:fragment>
                 </p:dataList>
             </h:form>
-            <!-- "facets unavailable" message: -->
+            <!-- "facets temporarily unavailable" message: -->
             <h:form id="facetsUnavailable" rendered="#{SearchIncludeFragment.facetsDisabled}">
                 <div class="clearfix">
-                    <h:outputText value="Facets temporarily unavailable" styleClass="facetTypeLink"/>
+                    <h:outputText value="#{bundle['dataverse.results.solrFacetsDisabled']}" styleClass="facetTypeLink"/>
                 </div>
             </h:form>
         </div>

From 59bc5ad1eb9f5e2cb05182d4c6320d20bbc28e65 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 25 Oct 2023 15:59:42 -0400
Subject: [PATCH 626/922] a couple of places in the api where we can save solr
 the trouble of generating facets. #9635

---
 src/main/java/edu/harvard/iq/dataverse/api/Search.java    | 4 +++-
 .../edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java | 8 ++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
index c760534ca7b..71e2865ca4d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
@@ -157,7 +157,9 @@ public Response search(
                         numResultsPerPage,
                         true, //SEK get query entities always for search API additional Dataset Information 6300  12/6/2019
                         geoPoint,
-                        geoRadius
+                        geoRadius,
+                        showFacets, // facets are expensive, no need to ask for them if not requested
+                        showRelevance // no need for highlights unless requested either
                 );
             } catch (SearchException ex) {
                 Throwable cause = ex;
diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java
index e9898031343..0a64f42d840 100644
--- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java
+++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java
@@ -39,7 +39,6 @@
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.QueryParam;
-import jakarta.ws.rs.DefaultValue;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.core.Context;
 
@@ -226,7 +225,12 @@ private SolrQueryResponse getTotalCountsFromSolr(DataverseRequest dataverseReque
                     //SearchFields.RELEASE_OR_CREATE_DATE, SortBy.DESCENDING,
                     0, //paginationStart,
                     true, // dataRelatedToMe
-                    SearchConstants.NUM_SOLR_DOCS_TO_RETRIEVE //10 // SearchFields.NUM_SOLR_DOCS_TO_RETRIEVE
+                    SearchConstants.NUM_SOLR_DOCS_TO_RETRIEVE, //10 // SearchFields.NUM_SOLR_DOCS_TO_RETRIEVE
+                    true, 
+                    null,
+                    null,
+                    false, // no need to request facets here ...
+                    false  // ... same for highlights
             );
         } catch (SearchException ex) {
             logger.severe("Search for total counts failed with filter query");

From 368382974cc641b46eb71026352a968b72e138be Mon Sep 17 00:00:00 2001
From: gwen <Gwenael.doux@cirad.fr>
Date: Thu, 26 Oct 2023 11:18:00 +0200
Subject: [PATCH 627/922] Fixed bad url

Fixed url for "editing files in another user repository"
---
 doc/sphinx-guides/source/developers/documentation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst
index f0729c59dcf..78b0970aaa5 100755
--- a/doc/sphinx-guides/source/developers/documentation.rst
+++ b/doc/sphinx-guides/source/developers/documentation.rst
@@ -8,7 +8,7 @@ Writing Documentation
 Quick Fix
 -----------
 
-If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following https://help.github.com/en/articles/editing-files-in-another-users-repository
+If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files#editing-files-in-another-users-repository
 
 - Navigate to https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source where you will see folders for each of the guides: `admin`_, `api`_, `developers`_, `installation`_, `style`_, `user`_.
 - Find the file you want to edit under one of the folders above.

From 2dfcea47dd26e120b45485f9db156e242747242f Mon Sep 17 00:00:00 2001
From: gwen <Gwenael.doux@cirad.fr>
Date: Thu, 26 Oct 2023 11:38:48 +0200
Subject: [PATCH 628/922] specify the extension case for XLSX files

---
 doc/sphinx-guides/source/installation/config.rst | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index ce8876b012c..1cc26debc64 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -3065,12 +3065,18 @@ You can override this global setting on a per-format basis for the following for
 - SAV
 - Rdata
 - CSV
-- XLSX
+- XLSX (in lower-case)
 
-For example, if you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB, use this setting:
+For example :
+
+* if you want your Dataverse installation to not attempt to ingest Rdata files larger than 1 MB, use this setting:
 
 ``curl -X PUT -d 1000000 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit:Rdata``
 
+* if you want your Dataverse installation to not attempt to ingest XLSX files at all, use this setting:
+
+``curl -X PUT -d 0 http://localhost:8080/api/admin/settings/:TabularIngestSizeLimit:xlsx``
+
 :ZipUploadFilesLimit
 ++++++++++++++++++++
 

From 25c180a81fdf9a84d1a13e67889f191cf85fa988 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 26 Oct 2023 10:55:25 -0400
Subject: [PATCH 629/922] guide entry, etc. #9635

---
 doc/release-notes/9635-solr-improvements.md   |  4 ++
 .../source/installation/prerequisites.rst     | 19 +++++++++
 .../search/SearchIncludeFragment.java         | 42 ++++++-------------
 .../dataverse/search/SearchServiceBean.java   | 15 +++----
 4 files changed, 40 insertions(+), 40 deletions(-)
 create mode 100644 doc/release-notes/9635-solr-improvements.md

diff --git a/doc/release-notes/9635-solr-improvements.md b/doc/release-notes/9635-solr-improvements.md
new file mode 100644
index 00000000000..ad55ee3afe6
--- /dev/null
+++ b/doc/release-notes/9635-solr-improvements.md
@@ -0,0 +1,4 @@
+- As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
+
+Please see the "Installing Solr" section of the Installation Prerequisites guide.
+
diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst
index 1847f1b8f63..a14a5cd19ab 100644
--- a/doc/sphinx-guides/source/installation/prerequisites.rst
+++ b/doc/sphinx-guides/source/installation/prerequisites.rst
@@ -211,6 +211,25 @@ Finally, you need to tell Solr to create the core "collection1" on startup::
 
         echo "name=collection1" > /usr/local/solr/solr-9.3.0/server/solr/collection1/core.properties
 
+Dataverse collection ("dataverse") page uses Solr very heavily. On a busy instance this may cause the search engine to become the performance bottleneck, making these pages take increasingly longer to load, potentially affecting the overall performance of the application and/or causing Solr itself to crash. If this is observed on your instance, we recommend uncommenting the following lines in the ``<circuitBreaker ...>`` section of the ``solrconfig.xml`` file::
+
+  <str name="memEnabled">true</str>
+  <str name="memThreshold">75</str>
+
+and::
+
+  <str name="cpuEnabled">true</str>
+  <str name="cpuThreshold">75</str>
+
+This will activate Solr "circuit breaker" mechanisms that make it start dropping incoming requests with the HTTP code 503 when it starts experiencing load issues. As of Dataverse 6.1, the collection page will recognize this condition and display a customizeable message to the users informing them that the search engine is unavailable because of heavy load, with the assumption that the condition is transitive and suggesting that they try again later. This is still an inconvenience to the users, but still a more graceful handling of the problem, rather than letting the pages time out or causing crashes. You may need to experiment and adjust the threshold values defined in the lines above. 
+
+If this becomes a common issue, another temporary workaround an admin may choose to use is to enable the following setting::
+
+  curl -X PUT -d true "http://localhost:8080/api/admin/settings/:DisableSolrFacets"
+
+This will make the collection show the search results without the usual search facets on the left side of the page. Another customizeable message will be shown in that column informing the users that facets are temporarily unavailable. Generating these facets is more resource-intensive for Solr than the main search results themselves, so applying this measure will significantly reduce the load on the search engine. 
+
+
 Solr Init Script
 ================
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 6ba5817bcdd..400f10cc375 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -318,9 +318,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         selectedTypesList = new ArrayList<>();
         String[] parts = selectedTypesString.split(":");
         selectedTypesList.addAll(Arrays.asList(parts));
-        
-        logger.info("selected types list size: "+selectedTypesList.size());
-        
+                
         String[] arr = selectedTypesList.toArray(new String[selectedTypesList.size()]);
         selectedTypesHumanReadable = combine(arr, " OR ");
         if (!selectedTypesHumanReadable.isEmpty()) {
@@ -382,9 +380,9 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             // extra numbers. -- L.A. 10/16/2023
             
             // populate preview counts: https://redmine.hmdc.harvard.edu/issues/3560
-            previewCountbyType.put(BundleUtil.getStringFromBundle("dataverses"), -1L);
-            previewCountbyType.put(BundleUtil.getStringFromBundle("datasets"), -1L);
-            previewCountbyType.put(BundleUtil.getStringFromBundle("files"), -1L);
+            previewCountbyType.put(BundleUtil.getStringFromBundle("dataverses"), 0L);
+            previewCountbyType.put(BundleUtil.getStringFromBundle("datasets"), 0L);
+            previewCountbyType.put(BundleUtil.getStringFromBundle("files"), 0L);
             
             
             // This will populate the type facet counts for the types that are 
@@ -397,34 +395,18 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             
             if (selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable() && !isFacetsDisabled()) {
                 // If some types are NOT currently selected, we will need to 
-                // run another query to obtain the numbers of the unselected types:
+                // run a second search to obtain the numbers of the unselected types:
                 
                 List<String> filterQueriesFinalSecondPass = new ArrayList<>();
                 filterQueriesFinalSecondPass.addAll(filterQueries);
-                
-                List<String> selectedTypesListSecondPass = new ArrayList<>();
-                
-                // @todo: simplify this!
-                for (String dvObjectTypeLabel : previewCountbyType.keySet()) {
-                    if (previewCountbyType.get(dvObjectTypeLabel) == -1) {
-                        String dvObjectType = null;
-                        
-                        if (dvObjectTypeLabel.equals(BundleUtil.getStringFromBundle("dataverses"))) {
-                            dvObjectType = "dataverses";
-                        } else if (dvObjectTypeLabel.equals(BundleUtil.getStringFromBundle("datasets"))) {
-                            dvObjectType = "datasets";
-                        } else if (dvObjectTypeLabel.equals(BundleUtil.getStringFromBundle("files"))) {
-                            dvObjectType = "files";
-                        }
-                    
-                        if (dvObjectType != null) {
-                            logger.info("adding object type to the second pass query: "+dvObjectType);
-                            selectedTypesListSecondPass.add(dvObjectType);
-                        }
+                   
+                arr = new String[3 - selectedTypesList.size()];
+                int c = 0; 
+                for (String dvObjectType : Arrays.asList("dataverses", "datasets", "files")) {
+                    if (!selectedTypesList.contains(dvObjectType)) {
+                        arr[c++] = dvObjectType;
                     }
                 }
-                
-                arr = selectedTypesListSecondPass.toArray(new String[selectedTypesListSecondPass.size()]);
                 filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
 
                 solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, null, sortOrder.toString(), 0, onlyDataRelatedToMe, 1, false, null, null, false, false);
@@ -432,7 +414,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 if (solrQueryResponseSecondPass != null) {
 
                     if (solrQueryResponseSecondPass.hasError()) {
-                        logger.info(solrQueryResponseSecondPass.getError());
+                        logger.fine(solrQueryResponseSecondPass.getError());
                         setSolrErrorEncountered(true);
                     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 6e410488794..51bf3bee30b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -376,7 +376,7 @@ public SolrQueryResponse search(
 //        solrQuery.addNumericRangeFacet(SearchFields.PRODUCTION_DATE_YEAR_ONLY, citationYearRangeStart, citationYearRangeEnd, citationYearRangeSpan);
 //        solrQuery.addNumericRangeFacet(SearchFields.DISTRIBUTION_DATE_YEAR_ONLY, citationYearRangeStart, citationYearRangeEnd, citationYearRangeSpan);
         solrQuery.setRows(numResultsPerPage);
-        logger.info("Solr query:" + solrQuery);
+        logger.fine("Solr query:" + solrQuery);
 
         // -----------------------------------
         // Make the solr query
@@ -389,12 +389,7 @@ public SolrQueryResponse search(
         } catch (RemoteSolrException ex) {
             String messageFromSolr = ex.getLocalizedMessage();
             
-            logger.info("message from tye solr exception: "+messageFromSolr);
-            logger.info("code from the solr exception: "+ex.code());
-            
-            if (queryResponse != null) {
-                logger.info("return code: "+queryResponse.getStatus());
-            }
+            logger.fine("message from the solr exception: "+messageFromSolr+"; code: "+ex.code());
             
             SolrQueryResponse exceptionSolrQueryResponse = new SolrQueryResponse(solrQuery);
 
@@ -444,9 +439,9 @@ public SolrQueryResponse search(
         
         int statusCode = queryResponse.getStatus();
         
-        logger.info("status code of the query response: "+statusCode);
-        logger.info("_size from query response: "+queryResponse._size());
-        logger.info("qtime: "+queryResponse.getQTime());
+        logger.fine("status code of the query response: "+statusCode);
+        logger.fine("_size from query response: "+queryResponse._size());
+        logger.fine("qtime: "+queryResponse.getQTime());
 
         SolrDocumentList docs = queryResponse.getResults();
         List<SolrSearchResult> solrSearchResults = new ArrayList<>();

From abd07549e301bd3cf681b89ebab040e4a348fa91 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 26 Oct 2023 11:31:23 -0400
Subject: [PATCH 630/922] documented the "disable facets" setting in the config
 section of the guide as well. #9635

---
 doc/sphinx-guides/source/installation/config.rst | 16 +++++++++++++++-
 .../source/installation/prerequisites.rst        |  2 +-
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index ce8876b012c..edc241664a0 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -2868,7 +2868,6 @@ To enable setting file-level PIDs per collection::
 
 When :AllowEnablingFilePIDsPerCollection is true, setting File PIDs to be enabled/disabled for a given collection can be done via the Native API - see :ref:`collection-attributes-api` in the Native API Guide.
 
-
 .. _:IndependentHandleService:
 
 :IndependentHandleService
@@ -3103,6 +3102,21 @@ If ``:SolrFullTextIndexing`` is set to true, the content of files of any size wi
 
 ``curl -X PUT -d 314572800 http://localhost:8080/api/admin/settings/:SolrMaxFileSizeForFullTextIndexing``
 
+
+.. _:DisableSolrFacets:
+
+:DisableSolrFacets
+++++++++++++++++++
+
+Setting this to ``true`` will make the collection ("dataverse") page start showing search results without the usual search facets on the left side of the page. A message will be shown in that column informing the users that facets are temporarily unavailable. Generating the facets is more resource-intensive for Solr than the main search results themselves, so applying this measure will significantly reduce the load on the search engine when its performance becomes an issue.
+
+This setting can be used in combination with the "circuit breaker" mechanism on the Solr side (see the "Installing Solr" section of the Installation Prerequisites guide). An admin can choose to enable it, or even create an automated system for enabling it in response to Solr beginning to drop incoming requests with the HTTP code 503.
+
+To enable the setting::
+
+  curl -X PUT -d true "http://localhost:8080/api/admin/settings/:DisableSolrFacets"
+
+
 .. _:SignUpUrl:
 
 :SignUpUrl
diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst
index a14a5cd19ab..7dbdfd30e02 100644
--- a/doc/sphinx-guides/source/installation/prerequisites.rst
+++ b/doc/sphinx-guides/source/installation/prerequisites.rst
@@ -227,7 +227,7 @@ If this becomes a common issue, another temporary workaround an admin may choose
 
   curl -X PUT -d true "http://localhost:8080/api/admin/settings/:DisableSolrFacets"
 
-This will make the collection show the search results without the usual search facets on the left side of the page. Another customizeable message will be shown in that column informing the users that facets are temporarily unavailable. Generating these facets is more resource-intensive for Solr than the main search results themselves, so applying this measure will significantly reduce the load on the search engine. 
+This will make the collection page show the search results without the usual search facets on the left side of the page. Another customizeable message will be shown in that column informing the users that facets are temporarily unavailable. Generating these facets is more resource-intensive for Solr than the main search results themselves, so applying this measure will significantly reduce the load on the search engine. 
 
 
 Solr Init Script

From b9f77096eb378074bd58e0244e40194984097d5f Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 18 Oct 2023 16:15:54 -0400
Subject: [PATCH 631/922] disable ConfigCheckServiceTest #10024

Jenkins is failing. Maybe disabling this will help.

It was added in this PR: #9819
---
 .../harvard/iq/dataverse/settings/ConfigCheckServiceTest.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
index b031b9429c6..dad86e73d19 100644
--- a/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/settings/ConfigCheckServiceTest.java
@@ -15,7 +15,9 @@
 
 import static java.nio.file.attribute.PosixFilePermission.GROUP_READ;
 import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import org.junit.jupiter.api.Disabled;
 
+@Disabled
 class ConfigCheckServiceTest {
     
     @TempDir

From 1902dbd541cebca2d5ec4441582fe03cb6eaf010 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 26 Oct 2023 14:21:53 -0400
Subject: [PATCH 632/922] replace 404 with accurate list of team members #9267

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d40e5f228f7..f81a8f86fd1 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 Dataverse&#174;  
 ===============
 
-Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).
+Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).
 
 [dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more.
 

From 60ed3a5ea24e4d091f1550ef0f10861b30df0b88 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 27 Oct 2023 12:07:47 -0400
Subject: [PATCH 633/922] label is optional

---
 .../harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index 1b61a9c9970..9bbc445ea9c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -1818,10 +1818,12 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f
 
         // labl
         if ((vm == null || !vm.containsKey("label"))) {
-            xmlw.writeStartElement("labl");
-            writeAttribute(xmlw, "level", "variable");
-            xmlw.writeCharacters(dvar.getString("label"));
-            xmlw.writeEndElement(); //labl
+            if(dvar.containsKey("label")) {
+                xmlw.writeStartElement("labl");
+                writeAttribute(xmlw, "level", "variable");
+                xmlw.writeCharacters(dvar.getString("label"));
+                xmlw.writeEndElement(); //labl
+            }
         } else if (vm != null && vm.containsKey("label")) {
             xmlw.writeStartElement("labl");
             writeAttribute(xmlw, "level", "variable");

From 22bdb303d4e669b7fd5dad2623d430743f8dc320 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Wed, 25 Oct 2023 14:51:05 -0400
Subject: [PATCH 634/922] handle missing frequency (QDR has 2 examples in the
 prod db)

---
 .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 65fe749e554..cfc266f2ba7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -796,9 +796,10 @@ private static JsonArrayBuilder jsonCatStat(Collection<VariableCategory> catStat
             JsonObjectBuilder catStatObj = Json.createObjectBuilder();
             catStatObj.add("label", stat.getLabel())
                       .add("value", stat.getValue())
-                      .add("isMissing", stat.isMissing())
-                      .add("frequency", stat.getFrequency())
-                    ;
+                      .add("isMissing", stat.isMissing());
+            if(stat.getFrequency()!=null){
+                catStatObj.add("frequency", stat.getFrequency());
+            }
             catArr.add(catStatObj);
         }
         return catArr;

From 8b1ab1d462cb143d77e1f1825c7b9386cc2552ce Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 27 Oct 2023 14:39:46 -0400
Subject: [PATCH 635/922] delete redundant test

---
 .../java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index 9bbc445ea9c..9a689f7a4ed 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -1824,7 +1824,7 @@ private static void createVarDDI(XMLStreamWriter xmlw, JsonObject dvar, String f
                 xmlw.writeCharacters(dvar.getString("label"));
                 xmlw.writeEndElement(); //labl
             }
-        } else if (vm != null && vm.containsKey("label")) {
+        } else {
             xmlw.writeStartElement("labl");
             writeAttribute(xmlw, "level", "variable");
             xmlw.writeCharacters(vm.getString("label"));

From a5bc9a1933cc25f3394f57ba93e7f6330d48023a Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Fri, 27 Oct 2023 23:50:26 +0200
Subject: [PATCH 636/922] Use JsonUtil.getJsonObject in AbstractApiBean

This fixes #10054.
Like before, JsonException may still be thrown.
Since this is a RuntimeException, I only mention it in the Javadoc.
---
 .../iq/dataverse/api/AbstractApiBean.java     | 22 +++++++++----------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
index 0a0861fa1c9..027f9e0fcb1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
@@ -47,9 +47,9 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.json.JsonParser;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean;
-import java.io.StringReader;
 import java.net.URI;
 import java.util.Arrays;
 import java.util.Collections;
@@ -62,9 +62,9 @@
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonArrayBuilder;
+import jakarta.json.JsonException;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
-import jakarta.json.JsonReader;
 import jakarta.json.JsonValue;
 import jakarta.json.JsonValue.ValueType;
 import jakarta.persistence.EntityManager;
@@ -132,23 +132,21 @@ public Response refineResponse( String message ) {
          * In the common case of the wrapped response being of type JSON,
          * return the message field it has (if any).
          * @return the content of a message field, or {@code null}.
+         * @throws JsonException when JSON parsing fails.
          */
         String getWrappedMessageWhenJson() {
             if ( response.getMediaType().equals(MediaType.APPLICATION_JSON_TYPE) ) {
                 Object entity = response.getEntity();
                 if ( entity == null ) return null;
 
-                String json = entity.toString();
-                try ( StringReader rdr = new StringReader(json) ){
-                    JsonReader jrdr = Json.createReader(rdr);
-                    JsonObject obj = jrdr.readObject();
-                    if ( obj.containsKey("message") ) {
-                        JsonValue message = obj.get("message");
-                        return message.getValueType() == ValueType.STRING ? obj.getString("message") : message.toString();
-                    } else {
-                        return null;
-                    }
+                JsonObject obj = JsonUtil.getJsonObject(entity.toString());
+                if ( obj.containsKey("message") ) {
+                    JsonValue message = obj.get("message");
+                    return message.getValueType() == ValueType.STRING ? obj.getString("message") : message.toString();
+                } else {
+                    return null;
                 }
+
             } else {
                 return null;
             }

From 0fabe3ead61aa47bddb385f92d6b69474dd82668 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 00:23:59 +0200
Subject: [PATCH 637/922] Remove Gson from JsonUtil

---
 .../iq/dataverse/util/json/JsonUtil.java      | 22 +++++++++----------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index cf8b64520de..2c780fa9417 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -1,9 +1,9 @@
 package edu.harvard.iq.dataverse.util.json;
 
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonObject;
-
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.HashMap;
@@ -11,6 +11,7 @@
 import java.util.logging.Logger;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
+import jakarta.json.JsonObject;
 import jakarta.json.JsonReader;
 import jakarta.json.JsonWriter;
 import jakarta.json.JsonWriterFactory;
@@ -26,11 +27,8 @@ public class JsonUtil {
      */
     public static String prettyPrint(String jsonString) {
         try {
-            com.google.gson.JsonParser jsonParser = new com.google.gson.JsonParser();
-            JsonObject jsonObject = jsonParser.parse(jsonString).getAsJsonObject();
-            Gson gson = new GsonBuilder().setPrettyPrinting().create();
-            String prettyJson = gson.toJson(jsonObject);
-            return prettyJson;
+            JsonObject jsonObject = getJsonObject(jsonString);
+            return prettyPrint(jsonObject);
         } catch (Exception ex) {
             logger.info("Returning original string due to exception: " + ex);
             return jsonString;
@@ -48,7 +46,7 @@ public static String prettyPrint(JsonArray jsonArray) {
         return stringWriter.toString();
     }
 
-    public static String prettyPrint(jakarta.json.JsonObject jsonObject) {
+    public static String prettyPrint(JsonObject jsonObject) {
         Map<String, Boolean> config = new HashMap<>();
         config.put(JsonGenerator.PRETTY_PRINTING, true);
         JsonWriterFactory jsonWriterFactory = Json.createWriterFactory(config);
@@ -59,7 +57,7 @@ public static String prettyPrint(jakarta.json.JsonObject jsonObject) {
         return stringWriter.toString();
     }
     
-    public static jakarta.json.JsonObject getJsonObject(String serializedJson) {
+    public static JsonObject getJsonObject(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
             try (JsonReader jsonReader = Json.createReader(rdr)) {
                 return jsonReader.readObject();
@@ -67,7 +65,7 @@ public static jakarta.json.JsonObject getJsonObject(String serializedJson) {
         }
     }
     
-    public static jakarta.json.JsonArray getJsonArray(String serializedJson) {
+    public static JsonArray getJsonArray(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
             try (JsonReader jsonReader = Json.createReader(rdr)) {
                 return jsonReader.readArray();

From 37725d99a0612c20a6af2210107e148c6b28398e Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 00:25:56 +0200
Subject: [PATCH 638/922] Get JSON objects from files and inputstreams

---
 .../iq/dataverse/util/json/JsonUtil.java      | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index 2c780fa9417..371eb7e543e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -65,6 +65,31 @@ public static JsonObject getJsonObject(String serializedJson) {
         }
     }
     
+    /**
+     * Return the contents of the {@link InputStream} as a JSON object.
+     * @param stream the input stream to read from
+     * @throws JsonException when parsing fails.
+     */
+    public static JsonObject getJsonObject(InputStream stream) {
+        try (JsonReader jsonReader = Json.createReader(stream)) {
+            return jsonReader.readObject();
+        }
+    }
+    
+    /**
+     * Return the contents of the file as a JSON object.
+     * @param fileName the name of the file to read from
+     * @throws FileNotFoundException when the file cannot be opened for reading
+     * @throws JsonException when parsing fails.
+     */
+    public static JsonObject getJsonObjectFromFile(String fileName) throws IOException {
+        try (FileReader rdr = new FileReader(fileName)) {
+            try (JsonReader jsonReader = Json.createReader(rdr)) {
+                return jsonReader.readObject();
+            }
+        }
+    }
+    
     public static JsonArray getJsonArray(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
             try (JsonReader jsonReader = Json.createReader(rdr)) {

From 82b5edf72d249d1dcd48209dcf19c0fad6bb375d Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 00:36:17 +0200
Subject: [PATCH 639/922] Remove superfluous exception declaration

---
 .../java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
index 6b48dbf8415..5f30de8e932 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
@@ -12,7 +12,6 @@
 import java.io.FileReader;
 import java.io.IOException;
 import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Iterator;
 import java.util.List;
@@ -135,7 +134,7 @@ public Response addUsageMetricsFromSushiReportAll(@PathParam("id") String id, @Q
 
     @POST
     @Path("{id}/updateCitationsForDataset")
-    public Response updateCitationsForDataset(@PathParam("id") String id) throws MalformedURLException, IOException {
+    public Response updateCitationsForDataset(@PathParam("id") String id) throws IOException {
         try {
             Dataset dataset = findDatasetOrDie(id);
             String persistentId = dataset.getGlobalId().toString();

From 69aa6735712532b87833a471ecb52f2f3ff33144 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 00:36:57 +0200
Subject: [PATCH 640/922] Use logger to log

---
 .../java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
index 5f30de8e932..e65d8da4b76 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
@@ -101,7 +101,7 @@ public Response addUsageMetricsFromSushiReport(@PathParam("id") String id, @Quer
             }
 
         } catch (IOException ex) {
-            System.out.print(ex.getMessage());
+            logger.log(Level.WARNING, ex.getMessage());
             return error(Status.BAD_REQUEST, "IOException: " + ex.getLocalizedMessage());
         }
         String msg = "Dummy Data has been added to dataset " + id;
@@ -125,7 +125,7 @@ public Response addUsageMetricsFromSushiReportAll(@PathParam("id") String id, @Q
             }
 
         } catch (IOException ex) {
-            System.out.print(ex.getMessage());
+            logger.log(Level.WARNING, ex.getMessage());
             return error(Status.BAD_REQUEST, "IOException: " + ex.getLocalizedMessage());
         }
         String msg = "Usage Metrics Data has been added to all datasets from file  " + reportOnDisk;

From 8cca38944664a4c8bd3b868f2dc1f36ab2841018 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 00:37:44 +0200
Subject: [PATCH 641/922] Delegate JSON parsing from input streams and files

See #10056
---
 .../iq/dataverse/api/MakeDataCountApi.java    | 35 ++++++++-----------
 1 file changed, 14 insertions(+), 21 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
index e65d8da4b76..25f3696fe6b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
@@ -8,8 +8,8 @@
 import edu.harvard.iq.dataverse.makedatacount.DatasetMetricsServiceBean;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
-import java.io.FileReader;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -82,23 +82,18 @@ public Response sendDataToHub() {
     @Path("{id}/addUsageMetricsFromSushiReport")
     public Response addUsageMetricsFromSushiReport(@PathParam("id") String id, @QueryParam("reportOnDisk") String reportOnDisk) {
 
-        JsonObject report;
-
-        try (FileReader reader = new FileReader(reportOnDisk)) {
-            report = Json.createReader(reader).readObject();
-            Dataset dataset;
-            try {
-                dataset = findDatasetOrDie(id);
-                List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, dataset);
-                if (!datasetMetrics.isEmpty()) {
-                    for (DatasetMetrics dm : datasetMetrics) {
-                        datasetMetricsService.save(dm);
-                    }
+        try {
+            JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk);
+            Dataset dataset = findDatasetOrDie(id);
+            List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, dataset);
+            if (!datasetMetrics.isEmpty()) {
+                for (DatasetMetrics dm : datasetMetrics) {
+                    datasetMetricsService.save(dm);
                 }
-            } catch (WrappedResponse ex) {
-                Logger.getLogger(MakeDataCountApi.class.getName()).log(Level.SEVERE, null, ex);
-                return error(Status.BAD_REQUEST, "Wrapped response: " + ex.getLocalizedMessage());
             }
+        } catch (WrappedResponse ex) {
+            logger.log(Level.SEVERE, null, ex);
+            return error(Status.BAD_REQUEST, "Wrapped response: " + ex.getLocalizedMessage());
 
         } catch (IOException ex) {
             logger.log(Level.WARNING, ex.getMessage());
@@ -112,10 +107,8 @@ public Response addUsageMetricsFromSushiReport(@PathParam("id") String id, @Quer
     @Path("/addUsageMetricsFromSushiReport")
     public Response addUsageMetricsFromSushiReportAll(@PathParam("id") String id, @QueryParam("reportOnDisk") String reportOnDisk) {
 
-        JsonObject report;
-
-        try (FileReader reader = new FileReader(reportOnDisk)) {
-            report = Json.createReader(reader).readObject();
+        try {
+            JsonObject report = JsonUtil.getJsonObjectFromFile(reportOnDisk);
 
             List<DatasetMetrics> datasetMetrics = datasetMetricsService.parseSushiReport(report, null);
             if (!datasetMetrics.isEmpty()) {
@@ -157,7 +150,7 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
                     logger.warning("Failed to get citations from " + url.toString());
                     return error(Status.fromStatusCode(status), "Failed to get citations from " + url.toString());
                 }
-                JsonObject report = Json.createReader(connection.getInputStream()).readObject();
+                JsonObject report = JsonUtil.getJsonObject(connection.getInputStream());
                 JsonObject links = report.getJsonObject("links");
                 JsonArray data = report.getJsonArray("data");
                 Iterator<JsonValue> iter = data.iterator();

From 1d1163b1be2fd35e66ec64cf5b91c2ee6d5d7bee Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 00:41:41 +0200
Subject: [PATCH 642/922] Get JSON object from stream using JsonUtil

---
 .../engine/command/impl/GetProvJsonCommand.java          | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java
index 2de2adff099..b068c0126dd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java
@@ -9,12 +9,12 @@
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.logging.Logger;
-import jakarta.json.Json;
 import jakarta.json.JsonObject;
-import jakarta.json.JsonReader;
 
 @RequiredPermissions(Permission.EditDataset)
 public class GetProvJsonCommand extends AbstractCommand<JsonObject> {
@@ -37,9 +37,8 @@ public JsonObject execute(CommandContext ctxt) throws CommandException {
             StorageIO<DataFile> dataAccess = dataFile.getStorageIO();
             InputStream inputStream = dataAccess.getAuxFileAsInputStream(provJsonExtension);
             JsonObject jsonObject = null;
-            if(null != inputStream) {
-                JsonReader jsonReader = Json.createReader(inputStream);
-                jsonObject = jsonReader.readObject();
+            if (null != inputStream) {
+                jsonObject = JsonUtil.getJsonObject(inputStream);
             }
             return jsonObject;
         } catch (IOException ex) {

From ff79e205fcb681e1a6487dd3136bb415a0992487 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 01:23:48 +0200
Subject: [PATCH 643/922] Expect 4 space indentation

This may become flaky if the indentation is dependent on implementation.
GSON apparently keeps empty objects in one line
and uses two spaces for indentation,
whereas I see slightly different outputs.
---
 .../harvard/iq/dataverse/util/json/JsonUtilTest.java   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonUtilTest.java
index 725862db7ba..3e4f9a690d2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonUtilTest.java
@@ -3,17 +3,17 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import org.junit.jupiter.api.Test;
 
-public class JsonUtilTest {
+class JsonUtilTest {
 
     @Test
-    public void testPrettyPrint() {
-        JsonUtil jsonUtil = new JsonUtil();
+    void testPrettyPrint() {
         String nullString = null;
         assertEquals(null, JsonUtil.prettyPrint(nullString));
         assertEquals("", JsonUtil.prettyPrint(""));
         assertEquals("junk", JsonUtil.prettyPrint("junk"));
-        assertEquals("{}", JsonUtil.prettyPrint("{}"));
-        assertEquals("{\n" + "  \"foo\": \"bar\"\n" + "}", JsonUtil.prettyPrint("{\"foo\": \"bar\"}"));
+        assertEquals("{\n}", JsonUtil.prettyPrint("{}"));
+        assertEquals("[\n    \"junk\"\n]", JsonUtil.prettyPrint("[\"junk\"]"));
+        assertEquals("{\n" + "    \"foo\": \"bar\"\n" + "}", JsonUtil.prettyPrint("{\"foo\": \"bar\"}"));
     }
 
 }

From 91e572623e4020c72744f1cb4c10f78aaef16518 Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 01:24:47 +0200
Subject: [PATCH 644/922] Add private constructor to JsonUtil

It only has static methods and should not be instantiated.
---
 src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index 371eb7e543e..d50cf6f124e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -21,6 +21,8 @@ public class JsonUtil {
 
     private static final Logger logger = Logger.getLogger(JsonUtil.class.getCanonicalName());
 
+    private JsonUtil() {}
+
     /**
      * Make an attempt at pretty printing a String but will return the original
      * string if it isn't JSON or if there is any exception.

From 785964df44317df84618bf217d479c41fdb6f46a Mon Sep 17 00:00:00 2001
From: bencomp <ben@companjen.name>
Date: Sat, 28 Oct 2023 01:25:25 +0200
Subject: [PATCH 645/922] Pretty-print JSON arrays from strings too

---
 .../java/edu/harvard/iq/dataverse/util/json/JsonUtil.java  | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index d50cf6f124e..8ea0e6f0ace 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -29,8 +29,11 @@ private JsonUtil() {}
      */
     public static String prettyPrint(String jsonString) {
         try {
-            JsonObject jsonObject = getJsonObject(jsonString);
-            return prettyPrint(jsonObject);
+            if (jsonString.trim().startsWith("{")) {
+                return prettyPrint(getJsonObject(jsonString));
+            } else {
+                return prettyPrint(getJsonArray(jsonString));
+            }
         } catch (Exception ex) {
             logger.info("Returning original string due to exception: " + ex);
             return jsonString;

From a4097a32998acfba5755f6103de86281c971b9a4 Mon Sep 17 00:00:00 2001
From: vidyaa18 <2010030534@klh.edu.in>
Date: Mon, 30 Oct 2023 00:33:30 +0530
Subject: [PATCH 646/922] made logo available

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d40e5f228f7..c3b8d9b9631 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ We love contributors! Please see our [Contributing Guide][] for ways you can hel
 
 Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States.
 
-[![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org)
+[![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg "Dataverse Project")](http://dataverse.org)
 
 [![API Test Status](https://jenkins.dataverse.org/buildStatus/icon?job=IQSS-dataverse-develop&subject=API%20Test%20Status)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/)
 [![API Test Coverage](https://img.shields.io/jenkins/coverage/jacoco?jobUrl=https%3A%2F%2Fjenkins.dataverse.org%2Fjob%2FIQSS-dataverse-develop&label=API%20Test%20Coverage)](https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/coverage-it/index.html)

From 47ea303562bf143d36edd66d035286ab74df6b9d Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Mon, 30 Oct 2023 08:48:05 +0100
Subject: [PATCH 647/922] Create InputStreams in try-with-resources

JsonUtil.getJsonObject closes the Readers, but not the InputStream.
It is the caller's responsibility to close the InputStream properly.
---
 .../harvard/iq/dataverse/api/MakeDataCountApi.java    |  6 +++++-
 .../engine/command/impl/GetProvJsonCommand.java       | 11 ++++++-----
 .../edu/harvard/iq/dataverse/util/json/JsonUtil.java  |  2 ++
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
index 25f3696fe6b..b2696757220 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
@@ -11,6 +11,7 @@
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Iterator;
@@ -150,7 +151,10 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
                     logger.warning("Failed to get citations from " + url.toString());
                     return error(Status.fromStatusCode(status), "Failed to get citations from " + url.toString());
                 }
-                JsonObject report = JsonUtil.getJsonObject(connection.getInputStream());
+                JsonObject report;
+                try (InputStream inStream = connection.getInputStream()) {
+                    report = JsonUtil.getJsonObject(inStream);
+                }
                 JsonObject links = report.getJsonObject("links");
                 JsonArray data = report.getJsonArray("data");
                 Iterator<JsonValue> iter = data.iterator();
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java
index b068c0126dd..b98cd70a4da 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetProvJsonCommand.java
@@ -35,12 +35,13 @@ public JsonObject execute(CommandContext ctxt) throws CommandException {
 
         try {
             StorageIO<DataFile> dataAccess = dataFile.getStorageIO();
-            InputStream inputStream = dataAccess.getAuxFileAsInputStream(provJsonExtension);
-            JsonObject jsonObject = null;
-            if (null != inputStream) {
-                jsonObject = JsonUtil.getJsonObject(inputStream);
+            try (InputStream inputStream = dataAccess.getAuxFileAsInputStream(provJsonExtension)) {
+                JsonObject jsonObject = null;
+                if (null != inputStream) {
+                    jsonObject = JsonUtil.getJsonObject(inputStream);
+                }
+                return jsonObject;
             }
-            return jsonObject;
         } catch (IOException ex) {
             String error = "Exception caught in DataAccess.getStorageIO(dataFile) getting file. Error: " + ex;
             throw new IllegalCommandException(error, this);
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index 8ea0e6f0ace..c1a20bf4c87 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -72,6 +72,8 @@ public static JsonObject getJsonObject(String serializedJson) {
     
     /**
      * Return the contents of the {@link InputStream} as a JSON object.
+     *
+     * The caller of this method is responsible for closing the provided stream.
      * @param stream the input stream to read from
      * @throws JsonException when parsing fails.
      */

From 235c0387921a79d1f919b97567666341119c5cae Mon Sep 17 00:00:00 2001
From: Ben Companjen <ben@companjen.name>
Date: Mon, 30 Oct 2023 09:52:27 +0100
Subject: [PATCH 648/922] Complete the Javadoc docs for JsonUtil.getJsonX

---
 .../iq/dataverse/util/json/JsonUtil.java      | 35 ++++++++++++++++---
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
index c1a20bf4c87..72a1cd2e1eb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java
@@ -11,6 +11,7 @@
 import java.util.logging.Logger;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
+import jakarta.json.JsonException;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonReader;
 import jakarta.json.JsonWriter;
@@ -61,7 +62,17 @@ public static String prettyPrint(JsonObject jsonObject) {
         }
         return stringWriter.toString();
     }
-    
+
+    /**
+     * Return the contents of the string as a JSON object.
+     * This method closes its resources when an exception occurs, but does
+     * not catch any exceptions.
+     * @param serializedJson the JSON object serialized as a {@code String}
+     * @throws JsonException when parsing fails.
+     * @see #getJsonObject(InputStream)
+     * @see #getJsonObjectFromFile(String)
+     * @see #getJsonArray(String)
+     */
     public static JsonObject getJsonObject(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
             try (JsonReader jsonReader = Json.createReader(rdr)) {
@@ -69,25 +80,33 @@ public static JsonObject getJsonObject(String serializedJson) {
             }
         }
     }
-    
+
     /**
      * Return the contents of the {@link InputStream} as a JSON object.
      *
+     * This method closes its resources when an exception occurs, but does
+     * not catch any exceptions.
      * The caller of this method is responsible for closing the provided stream.
      * @param stream the input stream to read from
      * @throws JsonException when parsing fails.
+     * @see #getJsonObject(String)
+     * @see #getJsonObjectFromFile(String)
      */
     public static JsonObject getJsonObject(InputStream stream) {
         try (JsonReader jsonReader = Json.createReader(stream)) {
             return jsonReader.readObject();
         }
     }
-    
+
     /**
      * Return the contents of the file as a JSON object.
+     * This method closes its resources when an exception occurs, but does
+     * not catch any exceptions.
      * @param fileName the name of the file to read from
      * @throws FileNotFoundException when the file cannot be opened for reading
      * @throws JsonException when parsing fails.
+     * @see #getJsonObject(String)
+     * @see #getJsonObject(InputStream)
      */
     public static JsonObject getJsonObjectFromFile(String fileName) throws IOException {
         try (FileReader rdr = new FileReader(fileName)) {
@@ -96,7 +115,15 @@ public static JsonObject getJsonObjectFromFile(String fileName) throws IOExcepti
             }
         }
     }
-    
+
+    /**
+     * Return the contents of the string as a JSON array.
+     * This method closes its resources when an exception occurs, but does
+     * not catch any exceptions.
+     * @param serializedJson the JSON array serialized as a {@code String}
+     * @throws JsonException when parsing fails.
+     * @see #getJsonObject(String)
+     */
     public static JsonArray getJsonArray(String serializedJson) {
         try (StringReader rdr = new StringReader(serializedJson)) {
             try (JsonReader jsonReader = Json.createReader(rdr)) {

From c43ca37235adfb038af7dbf9beace8e64868035a Mon Sep 17 00:00:00 2001
From: Yamil Suarez <github@yamil.com>
Date: Mon, 30 Oct 2023 08:50:25 -0400
Subject: [PATCH 649/922] Update issue template to set a default label

---
 .github/ISSUE_TEMPLATE/bug_report.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index b297dfc4ee8..7e6995d76d9 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -3,7 +3,7 @@ name: Bug report
 about: Did you encounter something unexpected or incorrect in the Dataverse software?
   We'd like to hear about it!
 title: ''
-labels: ''
+labels: 'Type: Bug'
 assignees: ''
 
 ---

From bec806af94fa5e738c2e029f9b928d6ac5b2fc0a Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 30 Oct 2023 10:42:26 -0400
Subject: [PATCH 650/922] (the "disable facets" switch should also cover the
 search facets on the dataset page #9635)

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index fc18257196d..d79d0acfdb6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -822,6 +822,12 @@ public List<FacetLabel> getFileTagsFacetLabels() {
      * @return boolean
      */
     public boolean isThisVersionSearchable() {
+        // Just like on the collection page, facets on the Dataset page can be
+        // disabled instance-wide by an admin:
+        if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacets, false)) {
+            return false;
+        }
+        
         SolrQuery solrQuery = new SolrQuery();
 
         solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, workingVersion.getDataset().getId().toString()));

From 904898979b7601d2f6a6e916fa489bf5c9a60316 Mon Sep 17 00:00:00 2001
From: Gustavo Durand <scolapasta+github@gmail.com>
Date: Mon, 30 Oct 2023 16:21:25 -0400
Subject: [PATCH 651/922] Update appendix.rst

---
 doc/sphinx-guides/source/user/appendix.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/user/appendix.rst b/doc/sphinx-guides/source/user/appendix.rst
index 7d60054ae17..e36b40a4110 100755
--- a/doc/sphinx-guides/source/user/appendix.rst
+++ b/doc/sphinx-guides/source/user/appendix.rst
@@ -37,7 +37,7 @@ Experimental Metadata
 Unlike supported metadata, experimental metadata is not enabled by default in a new Dataverse installation. Feedback via any `channel <https://dataverse.org/contact>`_ is welcome!
 
 - `CodeMeta Software Metadata <https://docs.google.com/spreadsheets/d/e/2PACX-1vTE-aSW0J7UQ0prYq8rP_P_AWVtqhyv46aJu9uPszpa9_UuOWRsyFjbWFDnCd7us7PSIpW7Qg2KwZ8v/pub>`__: based on the `CodeMeta Software Metadata Schema, version 2.0 <https://codemeta.github.io/terms/>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/codemeta.tsv>`__)
-- `Computational Workflow Metadata <https://docs.google.com/spreadsheets/d/13HP-jI_cwLDHBetn9UKTREPJ_F4iHdAvhjmlvmYdSSw/edit#gid=447508596>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/computationalworkflow.tsv>`__): adapted from `Bioschemas Computational Workflow Profile, version 1.0 <https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE>`__ and `Codemeta <https://codemeta.github.io/terms/>`__.
+- `Computational Workflow Metadata <https://docs.google.com/spreadsheets/d/13HP-jI_cwLDHBetn9UKTREPJ_F4iHdAvhjmlvmYdSSw/edit#gid=447508596>`__ (`see .tsv version <https://github.com/IQSS/dataverse/blob/master/scripts/api/data/metadatablocks/computational_workflow.tsv>`__): adapted from `Bioschemas Computational Workflow Profile, version 1.0 <https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE>`__ and `Codemeta <https://codemeta.github.io/terms/>`__.
 
 Please note: these custom metadata schemas are not included in the Solr schema for indexing by default, you will need
 to add them as necessary for your custom metadata blocks. See "Update the Solr Schema" in :doc:`../admin/metadatacustomization`.

From e02092c2b15a5a5de8565fa2ffe98e0ca2daa431 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 1 Nov 2023 11:50:04 -0400
Subject: [PATCH 652/922] Disabling quotas checks for direct uploads via API in
 the Create New Files command. Longer term solution coming as part of #8549.
 (#10080)

---
 .../engine/command/impl/CreateNewDataFilesCommand.java       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index ac701da1be9..0470f59b861 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -657,7 +657,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             if (newFileSize != null) {
                 fileSize = newFileSize;
             } else {
-                throw new CommandExecutionException("File size must be explicitly specified when creating DataFiles with Direct Upload", this);
+                // This is a direct upload via the API (DVUploader, etc.) 
+                //throw new CommandExecutionException("File size must be explicitly specified when creating DataFiles with Direct Upload", this);
             }
         }
         
@@ -696,7 +697,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             datafiles.add(datafile);
 
             // Update quota (may not be necessary in the context of direct upload - ?)
-            if (quota != null) {
+            if (fileSize > 0 && quota != null) {
                 quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
             }
             return CreateDataFileResult.success(fileName, finalType, datafiles);

From 8388a814cc46c9b1dcaeb93e4c482315260a2e8e Mon Sep 17 00:00:00 2001
From: Yamil Suarez <github@yamil.com>
Date: Wed, 1 Nov 2023 11:51:08 -0400
Subject: [PATCH 653/922] Add default label to request issue template

---
 .github/ISSUE_TEMPLATE/feature_request.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 7d5e0deea05..d6248537418 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea or new feature for the Dataverse software!
 title: 'Feature Request/Idea:'
-labels: ''
+labels: 'Type: Feature'
 assignees: ''
 
 ---

From bf5e28fc472d14914e4f2642fb8d1e4aaa85d33b Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 1 Nov 2023 14:45:25 -0400
Subject: [PATCH 654/922] some refactoring changes (#8549)

---
 .../impl/CreateNewDataFilesCommand.java       | 72 ++++++++++++-------
 1 file changed, 46 insertions(+), 26 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 0470f59b861..7a32e398b8f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -135,7 +135,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             }
         }
         String finalType = null;
-        
+        File newFile = null;    // this File will be used for a single-file, local (non-direct) upload
+        long fileSize = -1; 
+
+
         if (newStorageIdentifier == null) {
             if (getFilesTempDirectory() != null) {
                 try {
@@ -154,7 +157,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 // (note that "no size limit set" = "unlimited")
                 // (also note, that if this is a zip file, we'll be checking
                 // the size limit for each of the individual unpacked files)
-                Long fileSize = tempFile.toFile().length();
+                fileSize = tempFile.toFile().length();
                 if (fileSizeLimit != null && fileSize > fileSizeLimit) {
                     try {
                         tempFile.toFile().delete();
@@ -213,11 +216,11 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 }
 
                 DataFile datafile = null;
-                long fileSize = 0L; 
+                long uncompressedFileSize = -1; 
                 try {
                     uncompressedIn = new GZIPInputStream(new FileInputStream(tempFile.toFile()));
                     File unZippedTempFile = saveInputStreamInTempFile(uncompressedIn, fileSizeLimit, storageQuotaLimit);
-                    fileSize = unZippedTempFile.length();
+                    uncompressedFileSize = unZippedTempFile.length();
                     datafile = FileUtil.createSingleDataFile(version, unZippedTempFile, finalFileName, MIME_TYPE_UNDETERMINED_DEFAULT, ctxt.systemConfig().getFileFixityChecksumAlgorithm());
                 } catch (IOException | FileExceedsMaxSizeException | FileExceedsStorageQuotaException ioex) {
                     // it looks like we simply skip the file silently, if its uncompressed size
@@ -248,7 +251,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     datafiles.add(datafile);
                     // Update quota if present
                     if (quota != null) {
-                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
+                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + uncompressedFileSize);
                     }
                     return CreateDataFileResult.success(fileName, finalType, datafiles);
                 }
@@ -628,7 +631,35 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                     throw new CommandExecutionException("Failed to process uploaded BagIt file", ioex, this);
                 }
             }
+            
+            // These are the final File and its size that will be used to 
+            // add create a single Datafile: 
+            
+            newFile = tempFile.toFile();
+            fileSize = newFile.length();
+            
         } else {
+            // Direct upload.
+            
+            // Since this is a direct upload, and therefore no temp file associated 
+            // with it, we may, OR MAY NOT know the size of the file. If this is 
+            // a direct upload via the UI, the page must have already looked up 
+            // the size, after the client confirmed that the upload had completed. 
+            // (so that we can reject the upload here, i.e. before the user clicks
+            // save, if it's over the size limit or storage quota). However, if 
+            // this is a direct upload via the API, we will wait until the 
+            // upload is finalized in the saveAndAddFiles method to enforce the 
+            // limits. 
+            if (newFileSize != null) {
+                fileSize = newFileSize;
+                
+                // if the size is specified, and it's above the individual size 
+                // limit for this store, we can reject it now:
+                if (fileSizeLimit != null && fileSize > fileSizeLimit) {
+                    throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(fileSize), bytesToHumanReadable(fileSizeLimit)), this);
+                }
+            }
+            
             // Default to suppliedContentType if set or the overall undetermined default if a contenttype isn't supplied
             finalType = StringUtils.isBlank(suppliedContentType) ? FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT : suppliedContentType;
             String type = determineFileTypeByNameAndExtension(fileName);
@@ -639,34 +670,19 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 }
                 logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType);
             }
+            
+            
         }
+        
         // Finally, if none of the special cases above were applicable (or 
         // if we were unable to unpack an uploaded file, etc.), we'll just 
         // create and return a single DataFile:
-        File newFile = null;
-        long fileSize = -1; 
         
-        if (tempFile != null) {
-            newFile = tempFile.toFile();
-            fileSize = newFile.length();
-        } else {
-            // If this is a direct upload, and therefore no temp file associated 
-            // with it, the file size must be explicitly passed to the command 
-            // (note that direct upload relies on knowing the size of the file 
-            // that's being uploaded in advance).
-            if (newFileSize != null) {
-                fileSize = newFileSize;
-            } else {
-                // This is a direct upload via the API (DVUploader, etc.) 
-                //throw new CommandExecutionException("File size must be explicitly specified when creating DataFiles with Direct Upload", this);
-            }
-        }
         
         // We have already checked that this file does not exceed the individual size limit; 
         // but if we are processing it as is, as a single file, we need to check if 
         // its size does not go beyond the allocated storage quota (if specified):
         
-        
         if (storageQuotaLimit != null && fileSize > storageQuotaLimit) {
             if (newFile != null) {
                 // Remove the temp. file, if this is a non-direct upload. 
@@ -696,9 +712,13 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             }
             datafiles.add(datafile);
 
-            // Update quota (may not be necessary in the context of direct upload - ?)
-            if (fileSize > 0 && quota != null) {
-                quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
+            // Update quota (not necessary in the context of direct upload, will be done later)
+            // On a second thought, @todo: we should delay updating the storage size/quotas
+            // until the file is saved and finalized, for all upload cases!
+            if (newFile != null) {
+                if (fileSize > 0 && quota != null) {
+                    quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
+                }
             }
             return CreateDataFileResult.success(fileName, finalType, datafiles);
         }

From 3b19aebf7f913fb8a244ae8d8d6d9e151eb8fc2c Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 1 Nov 2023 15:31:38 -0400
Subject: [PATCH 655/922] replace link to 404 with link to team #9267

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 2333787d4b9..831dbfed5ff 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 Dataverse&#174;  
 ===============
 
-Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](https://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]).
+Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Dataverse team](https://dataverse.org/about) at the [Institute for Quantitative Social Science](https://iq.harvard.edu/) and the [Dataverse community][]).
 
 [dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more.
 

From 61abac1352f88a501bb0274d9689b0b3e5c8c017 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 1 Nov 2023 16:42:15 -0400
Subject: [PATCH 656/922] #9464 create json

---
 .../harvard/iq/dataverse/DataversePage.java   |   2 +
 .../iq/dataverse/DataverseServiceBean.java    | 211 ++++++++++++++++++
 2 files changed, 213 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
index 943a74327d5..12b7e41b3d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
@@ -597,6 +597,8 @@ public void updateOptionsRadio(Long mdbId, Long dsftId) {
 
 
     public String save() {
+                 String test = dataverseService.getCollectionDatasetSchema(dataverse.getId());
+                 System.out.print(test);
         List<DataverseFieldTypeInputLevel> listDFTIL = new ArrayList<>();
         if (editMode != null && ( editMode.equals(EditMode.INFO) || editMode.equals(EditMode.CREATE))) {
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 549b8310122..070fc20a5da 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -42,6 +42,7 @@
 import jakarta.persistence.NonUniqueResultException;
 import jakarta.persistence.PersistenceContext;
 import jakarta.persistence.TypedQuery;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.solr.client.solrj.SolrServerException;
 
 /**
@@ -80,6 +81,9 @@ public class DataverseServiceBean implements java.io.Serializable {
     @EJB
     PermissionServiceBean permissionService;
     
+    @EJB
+    DataverseFieldTypeInputLevelServiceBean dataverseFieldTypeInputLevelService;
+    
     @EJB
     SystemConfig systemConfig;
 
@@ -919,5 +923,212 @@ public List<Object[]> getDatasetTitlesWithinDataverse(Long dataverseId) {
         return em.createNativeQuery(cqString).getResultList();
     }
 
+        
+    public  String getCollectionDatasetSchema(Long dataverseId) {
+       
+        List<MetadataBlock> selectedBlocks = new ArrayList<>();
+        List<DatasetFieldType> requiredDSFT = new ArrayList<>();
+
+        Dataverse testDV = this.find(dataverseId);
+
+        while (!testDV.isMetadataBlockRoot()) {
+            if (testDV.getOwner() == null) {
+                break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value
+            }
+            testDV = testDV.getOwner();
+        }
+        
+        selectedBlocks.addAll(testDV.getMetadataBlocks());
+
+        for (MetadataBlock mdb : selectedBlocks) {
+            for (DatasetFieldType dsft : mdb.getDatasetFieldTypes()) {
+                if (!dsft.isChild()) {
+                    DataverseFieldTypeInputLevel dsfIl = dataverseFieldTypeInputLevelService.findByDataverseIdDatasetFieldTypeId(testDV.getId(), dsft.getId());
+                    if (dsfIl != null) {
+                        dsft.setRequiredDV(dsfIl.isRequired());
+                        dsft.setInclude(dsfIl.isInclude());
+                    } else {
+                        dsft.setRequiredDV(dsft.isRequired());
+                        dsft.setInclude(true);
+                    }
+                    if (dsft.isHasChildren()) {
+                        for (DatasetFieldType child : dsft.getChildDatasetFieldTypes()) {
+                            DataverseFieldTypeInputLevel dsfIlChild = dataverseFieldTypeInputLevelService.findByDataverseIdDatasetFieldTypeId(testDV.getId(), child.getId());
+                            if (dsfIlChild != null) {
+                                child.setRequiredDV(dsfIlChild.isRequired());
+                                child.setInclude(dsfIlChild.isInclude());
+                            } else {
+                                // in the case of conditionally required (child = true, parent = false)
+                                // we set this to false; i.e this is the default "don't override" value
+                                child.setRequiredDV(child.isRequired() && dsft.isRequired());
+                                child.setInclude(true);
+                            }
+                        }
+                    }
+                    if(dsft.isRequiredDV()){
+                        requiredDSFT.add(dsft);
+                    }
+                }
+            }            
+
+        }
+        
+        String reqMDBNames = "";
+        List<MetadataBlock> hasReqFields = new ArrayList<>();
+        String retval = datasetSchemaPreface;
+        for (MetadataBlock mdb : selectedBlocks) {
+            for (DatasetFieldType dsft : requiredDSFT) {
+                if (dsft.getMetadataBlock().equals(mdb)) {
+                    hasReqFields.add(mdb);
+                    if (!reqMDBNames.isEmpty()) reqMDBNames += ",";
+                    reqMDBNames += "\"" + mdb.getName() + "\"";
+                    break;
+                }
+            }
+        }
+        
+        for (MetadataBlock mdb : hasReqFields) {
+            retval += getCustomMDBSchema(mdb, requiredDSFT);
+        }
+        
+        retval += "\n}\n";
+        
+        retval += endOfjson.replace("blockNames", reqMDBNames);
+
+        return retval;
+    
+    } 
+    
+    private  String datasetSchemaPreface = 
+    "{\n" +
+    "    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n" +
+    "    \"$defs\": {\n" +
+    "    \"field\": {\n" + 
+    "        \"type\": \"object\",\n" +
+    "        \"required\": [\"typeClass\", \"multiple\", \"typeName\"],\n" +
+    "        \"properties\": {\n" + 
+    "            \"value\": {\n" +
+    "                \"anyOf\": [\n" +
+    "                    {\n" +
+    "                        \"type\": \"array\"\n" +
+    "                    },\n" +
+    "                    {\n" + 
+    "                        \"type\": \"string\"\n" +
+    "                    },\n" +
+    "                    {\n" +
+    "                        \"$ref\": \"#/$defs/field\"\n" +
+    "                    }\n" + 
+    "                ]\n" + 
+    "            },\n" + 
+    "            \"typeClass\": {\n" +
+    "                \"type\": \"string\"\n" +
+    "            }\n," +
+    "            \"multiple\": {\n" +
+    "                \"type\": \"boolean\"\n" +
+    "            },\n" +
+    "            \"typeName\": {\n" + 
+    "                \"type\": \"string\"\n" +
+    "            }\n" +
+    "        }\n" +
+    "    }\n" + 
+    "},\n" + 
+    "\"type\": \"object\",\n" +
+    "\"properties\": {\n" + 
+    "    \"datasetVersion\": {\n" + 
+    "        \"type\": \"object\",\n" +
+    "        \"properties\": {\n" + 
+    "           \"license\": {\n" + 
+    "                \"type\": \"object\",\n" + 
+    "                \"properties\": {\n" + 
+    "                    \"name\": {\n" +
+    "                        \"type\": \"string\"\n" + 
+    "                    },\n" + 
+    "                    \"uri\": {\n" + 
+    "                        \"type\": \"string\",\n" + 
+    "                        \"format\": \"uri\"\n" + 
+    "                   }\n" + 
+    "                },\n" + 
+    "                \"required\": [\"name\", \"uri\"]\n" + 
+    "            },\n" + 
+    "            \"metadataBlocks\": {\n" + 
+    "                \"type\": \"object\",\n" + 
+    "               \"properties\": {\n" +
+    ""  ;  
+
+
+    
+    private String reqValTemplate = "                                        {\n" +
+"                                            \"contains\": {\n" +
+"                                                \"properties\": {\n" +
+"                                                    \"typeName\": {\n" +
+"                                                        \"const\": \"reqFieldTypeName\"\n" +
+"                                                    }\n" +
+"                                                }\n" +
+"                                            }\n" +
+"                                        },";
+    
+    private String minItemsTemplate = "\n                                    \"minItems\": numMinItems,\n" +
+"                                    \"allOf\": [\n";
+    private String endOfReqVal = "                                    ]\n" +
+"                                }\n" +
+"                            },\n" +
+"                            \"required\": [\"fields\"]\n" +
+"                        },";
+    
+    private String endOfjson = ",\n" +
+"                    \"required\": [blockNames]\n" +
+"                }\n" +
+"            },\n" +
+"            \"required\": [\"license\", \"metadataBlocks\"]\n" +
+"        }\n" +
+"    },\n" +
+"    \"required\": [\"datasetVersion\"]\n" +
+"}\n";
+    
+    private String startOfMDB = "\"blockName\": {\n" +
+"                            \"type\": \"object\",\n" +
+"                            \"properties\": {\n" +
+"                                \"fields\": {\n" +
+"                                    \"type\": \"array\",\n" +
+"                                    \"items\": {\n" +
+"                                        \"$ref\": \"#/$defs/field\"\n" +
+"                                    },";
+    
     
+    private String getCustomMDBSchema (MetadataBlock mdb, List<DatasetFieldType> requiredDSFT){
+        String retval = "";
+        boolean mdbHasReqField = false;
+        int numReq = 0;
+        List<DatasetFieldType> requiredThisMDB = new ArrayList<>();
+        
+        for (DatasetFieldType dsft : requiredDSFT ){
+
+            if(dsft.getMetadataBlock().equals(mdb)){
+                numReq++;
+                mdbHasReqField = true;
+                requiredThisMDB.add(dsft);
+            }
+        }
+        if (mdbHasReqField){
+        retval  += startOfMDB.replace("blockName", mdb.getName());
+        
+        retval += minItemsTemplate.replace("numMinItems", Integer.toString(requiredThisMDB.size()));
+        int count = 0;
+        for (DatasetFieldType dsft:requiredThisMDB ){
+            count++;
+            String reqValImp = reqValTemplate.replace("reqFieldTypeName", dsft.getName());
+            if (count < requiredThisMDB.size()){
+                retval += reqValImp + "\n";
+            } else {
+               reqValImp = StringUtils.substring(reqValImp, 0, reqValImp.length() - 1);
+               retval += reqValImp+ "\n";
+               retval += endOfReqVal;
+            }            
+        }
+        
+        }
+
+        return retval;
+    }    
+            
 }

From 38f09f6e401d069956b56df94eb03437ca905a0e Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 2 Nov 2023 09:32:20 -0400
Subject: [PATCH 657/922] #9464 fix json schema formatting

---
 .../edu/harvard/iq/dataverse/DataverseServiceBean.java     | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 070fc20a5da..be2a97541c6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -991,7 +991,7 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
             retval += getCustomMDBSchema(mdb, requiredDSFT);
         }
         
-        retval += "\n}\n";
+        retval += "\n                     }";
         
         retval += endOfjson.replace("blockNames", reqMDBNames);
 
@@ -1022,7 +1022,7 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
     "            },\n" + 
     "            \"typeClass\": {\n" +
     "                \"type\": \"string\"\n" +
-    "            }\n," +
+    "            },\n" +
     "            \"multiple\": {\n" +
     "                \"type\": \"boolean\"\n" +
     "            },\n" +
@@ -1085,7 +1085,8 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
 "    \"required\": [\"datasetVersion\"]\n" +
 "}\n";
     
-    private String startOfMDB = "\"blockName\": {\n" +
+    private String startOfMDB = "" +
+"                           \"blockName\": {\n" +
 "                            \"type\": \"object\",\n" +
 "                            \"properties\": {\n" +
 "                                \"fields\": {\n" +

From 5ca4cc08b92b0fe0c4f44bcd3c8da2e32f5ebb47 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 2 Nov 2023 09:48:51 -0400
Subject: [PATCH 658/922] #9464 remove license from required

---
 .../java/edu/harvard/iq/dataverse/DataverseServiceBean.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index be2a97541c6..5942d4a8010 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -1079,7 +1079,7 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
 "                    \"required\": [blockNames]\n" +
 "                }\n" +
 "            },\n" +
-"            \"required\": [\"license\", \"metadataBlocks\"]\n" +
+"            \"required\": [\"metadataBlocks\"]\n" +
 "        }\n" +
 "    },\n" +
 "    \"required\": [\"datasetVersion\"]\n" +

From 2efdece6784533170733490928137e4c60c98d2d Mon Sep 17 00:00:00 2001
From: "Balazs E. Pataki" <pataki@gmail.com>
Date: Thu, 2 Nov 2023 16:16:51 +0100
Subject: [PATCH 659/922] Add gdcc/nginx:unstable image and dev_nginx container

Added a nginx container to the docker setup. This could be used for
various proxying stuff, but it is primarily added to work around a problem
with the IntelliJ Payara plugin. It doesn't allow remote redeployment in
case the Payara admin is served via HTTPS using a self-signed certificate,
which is the case of the default dataverse container installation.

This nginx configuration provides an HTTP endpoint at port 4849, and proxies
requests to the Payara admin console's HTTPS 4848 endpoint.

From the IntelliJ Payara plugin one has to specify the localhost 4849 port
(without SSL).
---
 docker-compose-dev.yml     |   9 +++++++++
 modules/nginx/Dockerfile   |   9 +++++++++
 modules/nginx/README.md    |   7 +++++++
 modules/nginx/default.conf |  12 ++++++++++++
 modules/nginx/img.png      | Bin 0 -> 71929 bytes
 pom.xml                    |   8 ++++++++
 6 files changed, 45 insertions(+)
 create mode 100644 modules/nginx/Dockerfile
 create mode 100644 modules/nginx/README.md
 create mode 100644 modules/nginx/default.conf
 create mode 100644 modules/nginx/img.png

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index ab44dbc1806..743b5d698d9 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -127,6 +127,15 @@ services:
     volumes:
       - './conf/keycloak/oidc-realm.json:/opt/keycloak/data/import/oidc-realm.json'
 
+  dev_nginx:
+    container_name: dev_nginx
+    image: gdcc/dev_nginx:unstable
+    ports:
+      - "4849:4849"
+    restart: always
+    networks:
+      - dataverse
+
 networks:
   dataverse:
     driver: bridge
diff --git a/modules/nginx/Dockerfile b/modules/nginx/Dockerfile
new file mode 100644
index 00000000000..3900076599f
--- /dev/null
+++ b/modules/nginx/Dockerfile
@@ -0,0 +1,9 @@
+FROM nginx:latest
+
+# Remove the default NGINX configuration file
+RUN rm /etc/nginx/conf.d/default.conf
+
+# Copy the contents of the local default.conf to the container
+COPY default.conf /etc/nginx/conf.d/
+
+EXPOSE 4849
\ No newline at end of file
diff --git a/modules/nginx/README.md b/modules/nginx/README.md
new file mode 100644
index 00000000000..9d2ff785577
--- /dev/null
+++ b/modules/nginx/README.md
@@ -0,0 +1,7 @@
+# nginx proxy
+
+nginx can be used to proxy various services at other ports/protocols from docker.
+
+Currently, this is used to work around a problem with the IntelliJ Payara plugin, which doesn't allow remote redeployment in case the Payara admin is served via HTTPS using a self-signed certificate, which is the case of the default dataverse container installation. This configuration provides an HTTP endpoint at port 4849, and proxies requests to the Payara admin console's HTTPS 4848 endpoint. From the IntelliJ Payara plugin one has to specify the localhost 4849 port (without SSL).
+
+![img.png](img.png)
diff --git a/modules/nginx/default.conf b/modules/nginx/default.conf
new file mode 100644
index 00000000000..8381a66c19a
--- /dev/null
+++ b/modules/nginx/default.conf
@@ -0,0 +1,12 @@
+server {
+    listen 4849;
+
+    # Make it big, so that .war files can be submitted
+    client_max_body_size 300M;
+
+    location / {
+        proxy_pass https://dataverse:4848;
+        proxy_ssl_verify off;
+        proxy_ssl_server_name on;
+    }
+}
diff --git a/modules/nginx/img.png b/modules/nginx/img.png
new file mode 100644
index 0000000000000000000000000000000000000000..278ee490b2d6d7c61b641cae4133b022c66098e9
GIT binary patch
literal 71929
zcmeF3Wmr_-7w=I}P+C-aXb`1429O335a}E`q@-s^B?rj?5hVr@l$7q2kcOc<q`QXh
zd&b{C?!7PWeRp5oc^=<z<{Z!Lv-jF-eZJpy!e6N=65gk}kA;OrsH`OS8Vl<-3iy|X
zcNh2z;*#Y#7S=l~WjSdrsL9sMohm2o5$?A;Y~S>run*2WA$$?`;>k?vhj)xKEK%-W
ziSAw$0&sWX+k?K}1P;!VPolTYQ|nJ)>brh@6sUuM<BI})i)hiYlpE1ey<O0Dz|Bd3
z3L`>^`9ZiGBVrA7b3WGC6-MW;XX3s7ozFm=_uq$<2=JK2I^F*L)d7Y$Fx%gAfKzE1
z|Mvq-6uXT#{%5OFH=@6eND0M`<m>-_nF67N20y@NCmRRt)u-CnKCh?q-IFH2=TIRk
zvYjjI*HtkTDp;1Yy{0bir$k!uW^$A6=2Q{3TYqx?P~&@dvU7v}tpCNf<agb_H^M}v
zj8rtQJ6(ua{$hh%4@}c_@w1J4?ID5gU|J3)Q+why3$uZ(rg}B>uau%B4NZKu%h9_5
zH?>TBPf>0t30*bu?K0Qp1YIKG*OYR{oos0wUUPn4)hDw)c~@%^Ws@-t#?bkY<qoEp
z{A2IK0mW|TfQu>D^Ytt*%QUk)S8{7DQ<o==H}zXJn+417u!*vny=^Dx|Bl&pG&72c
zouv5cWUldYfxIz$+%jIZYbDv)<apW*^@1pfqk^vY<alOIs7bb_Pjoe<r1}1{kH9ai
z1`bP<*Q*N8lvSK{L#|64%13mTPZwhfEz4A=N62gB{7&Wq0`&ckM!KD9w{>RyPG+3y
z#~dYmYDV*GYY#gPHu4HTWyDC<IFt=2SOLF|E3csD&4hUbXV=jfWbS8QLHXuJpN^y9
z4lpKG3EDb29&_$Ps>|<jK~Jcis?0=>$1NKB+mVMm1;pF-y}ORIlgC6xF25fw4{NG(
za@Xz2une@_&zZAIH148yuH${)IJwAM<e3cfexZVyN8hCAI}eoZ<Akh|+>_Y*h20&_
zhSc+%g{_vfQQ2#mcVmC}MXvzrRjjU@_0BU{+&@YkBaM}xzr)>li6-xeL0#<BI_m>Z
zDlhS@V#+CR>1}DR(6ca*w=K)2w9?7AMGX3UvoPTO^Zih!HyeX@(U+BtS3mPh`^1*L
z8ZV4{Tigg$n<q!fI*(_L`c+IoT4R1O4P}mZ6lq-KMi7o9R~gY_x%rLP=j9OL>9?|p
zx}W^rVb^D?_QnmBj#->(M-l$g0tsg$h5p*akHCHDm8e69$=_Z_7qeT5#b{&ExshF;
z!)IH6?mZvQcuDh|=7CIM|IFvDs>Nt4;6df#=ItFHG*b^1Rf`SBn6`uA4T4i1sx@a1
zaAS&SB@&3ISi;pXqZA+;w~Ba(M}Xo)`_B2DB&bVWl|4wF@tUiT%apoWS0CA|UjAhG
zNmEVqdxX(FwsGrZ$1XB;u!i!;tk15o%c#E1NL}OgQDL_-TdLMQ{a9WTEvw<1#Kz0-
zaj#-7uGm0FjFw6no`uybN$3GZx&&Pl*moMwakB(Vl_}@Ot2dv+1eRbk9%Dai(4p5W
z!i_nzn^Kqa#Kx9{ymP>`>Kam)GVZnw@^7XiF~&LBYauQ?2mV~KTzAlyw{!(edOz|e
zZQ|P}{j$W`-|Cv~Qh%)7smb%b+;1=MQ{k%4%&%JbpdCZMZj<Kc>4@7oDv5HBeHH8z
zM(dgFecnA3&@YXvO+Datv0YhsHAqyr^Rsf+0OY&ZWDw(Dl2CW9;5>$sy15jKE+{Un
znOHytT)T}RdZA&&lBa)inNT;QhR(*(qAQ;b+Qy66nzzIaORB|!o*aa3O3fT@_VA<f
z7Z?95{;>iT<#>-bLvs@SmNRf`?Z%o~q~+!&QoW2D*SL)2(>^A8Hr*k2r@j3*I<0X7
z(JAhmDpKlfDgFf&EF*-O-pSl45#oRLB9@Vrk~BX072{VW5@*4|5Z?k|_Li$H8K&Zi
z>~g%`5Puy8#;2Uu#$i5RidW})_Ot9Y%jb@VuftznJTsD^1uug0H{^H_qT5Ukl7;1?
z27&@O2wA!HCi^_&tARvWBGJsa>cwdBf^O`-AkL3+{+Wc71n8H-!AK{fdj%o0br~{B
z<IP_4?CuH<<t-U%+>WJmKBcs-a44FmG{jYj6`c!d78swdqQLe(@8jAU8aAXoRA{5s
zi`5Lip0qFg$$YclPFHC04FWH6L8M7c^nD(HOt}w5%XTN(7wF#umqb_L7Y>l0O<+&_
z$Z9BQ(Nvev-qh@C>d?zU*;H6!Z+n~56DkWif7F;IyLT3qEE|4Jp+1&8djV`4O3deC
zap0ItJapvNR{5Ao0uKX+(j#JqLHBlAFD=)w)4MPQEd&zSR;uD`GAS8k7)e!U6_%pe
zPJjTE$4s=pM0!LEvyZp2)P2c0s7O3J+K*klnDCv+Ri4f9Aeb?F5MhRN-nNDK0pVJc
z1rj51tjB^1xLl;OuN_1PA~iDk>UIu?xrJ=wY&?wJ2jsPdL7H<dq0*Nvn$dWSpTR|G
z0oSnVOLVt=a<dMJuHSR|`n8V7;GgaaWi8avdU@^Ap|c$seBkyrIkCWPENN$t0cr+$
zWXN36kjI#@Zf7YQj#7dDK-0Xu|E!&-e21}O$%KRnV}g^>M2wSCDepd-tv>jusL_{5
zU=``(lo-_v0`Il`g4*dy(KM{F)3o1uS0C(e<T0Wx$TfTG_xrHf-nR+Im#TlGWg#OF
zEkCfr6y70}mqNoxC+;nTUNSgIBN5rGFdd?FvX&3|3`z5<+PLbrtzC>hGYA_TttVKF
zolL&K;?Xq@q{m(eh3|VazJ(jt@xb%Xj_|!1vSqMhmCq0|-g33gYJsQgl~sn@5p(i<
zFu5tD4bCH8hYb$Gb-A4-MLywK=py%UcqC)kLdbGDb*hmK{uRNl*n{p+X(ZD$LkB;K
zN2W75sz?mWR<?F)si>bB)hr*Wc$lY*{LZvRs+sRtvuA0aOpcu0u(F5IBt_mDESLd)
z&0)xU5dl%2enZNu4_PGh`kv4ME@KkAML09l(8{OFHq!l3a!+{i-^}Bm9pi)b_+96M
z#1U|y;ttdfrm(mynfcTli6|0+tw_uH$Nm7W8b_kY+^u5TVH7(KhecNzG2HHW<Qj>`
zlpv!ND~z*W7;t6J@~7kJ?RI+_vpaQlCX<@ZBpOQ-P48^J?6rp4Q0)(o?SN6ucaxU%
z>6WC?u`lG*rJi33m4uI@7sQ})koAYH<oj|CLEs^}^cYzgwk1(8%3<ahtU`<`g}Qn2
z5Cz*v;`zpdhxF;*SAN-161q4@iO|q3P(_m7RIGlH`2AblR0L-rVmp#9`9rC79zAN`
zJl)+>FwuxVEm9(;E{kYGd@+)L18j$y`)B1m#jG4);x65t-$zrHM8X+_l?VgFwUiG^
zXk|j=2qVGxQ*Q~I28o=D+?ObM75ZYCk#@5NRz|W^w|EsEAPnR*L~w6+TznvTAsbDq
zKC5=N&IQIPmKQ<@a@R8Sfx8;=!a#LNXr?^UgM7$#5#iTb@3m|Y18bs*x81gQAe-f_
z&Xq4ZS^al1$weJN%&nrtT(HMtOzN#~gpYKxcx9uR9?C_Q`uhyLh9TE?I!eW0S+Flr
z8RQqiuTGt|5H@FwOTnwNW^K2c#!JKp3%J{4X-#equR?R1kZtvB==_Gf!iJjYMpq)&
zl@J~x%R0)Z@qZsWMDgziNZK4S&G53$r^Ap$AEXLWWX89X>40LBiZ2P9P#kXJ;~B-;
zG{*k<4Yo`sCpD7^9wx4v>_`*4N{CKj%Brh@Kl+x#;rcY@m_#l(Q4O5Y;8PaUGpH*8
z{}y}C68{UCLOM9`vE1XFAGpn{Eb>TeJ0;;=dvOQswN~-y%RxhluO>J7EdCo&!a^q5
zAGx?0Iaxo?5|N(E+*s=o4AL%%`d^53xE+isFKLjA_Z#|c{Y)czeE8=UF}Eaz1up37
z1NmK2e@i2}_2rsGK7{cxA8zacTsOM^(PG)OYq$1ohq}d(rqfQ*8rx@E76l@o*EcyU
zr(aJArUinT4#7hmZkLmIsS^oYbu$a;{0>Z*%<<|TCaI;*lX(zp%6wmWK19w421U^9
z=Qp#&n#7#^CV58I$g;&N*{VL&nE_^mzhuw=f87`FkRc8>g*Np=<X$K5qts-C(_r)n
ziN@jQWBsJn+MlA8eiV^4zIfkyX8^5RQv$9>4;UH_!pdNF<%N#h^bv4(uN&7S$!4;$
zmV+Mgyt#DEB{P-ty@og3TZ|IT>VYYxA>czTVq#(A5$eXU>7Jma`*~k0!G{VjNME%|
zI&?hiY1w*}StkaIT_-szmOUJRcn=b$$G9BNh$u=nX1^z3syVa}4|}rRh22pB0S`3n
zXT)rp1$fB$wFykdqE5evcXfBG^@{$@o#g_a19O7{t{gjgTu#q92n{rjXus^`8%&a7
zVU2MggT})kEg3XN^4|1#I_l^0*Hs`fW#@{q<5vo2N)i6=U*VhRn7Dz|8ScwGYE=`S
zdh^XLqC)M~wj3y+w$w4cOvsVY(MRYY+mIp%``)hmC6@p(&jZ@!Ng=z@3FGALhTkG5
znE@XiH>t*&C3}Tx?1j4&sp}`W{@&YlATuhw={9%Hsm(m~J_LITrtom>!G5F6i%WE{
zo*`&bC}&IMt1x$A7M6M1`Zme0iz(J(OmU^etUv%1O?Rp1<+oH?Mf;B#E9<}cJ-X7~
z{pa8#X4v1v5c5qjxwzH8f5oKJvj6|U|7}P>dq9H7|M&U$M;teNQeqAziUWILNAv-8
zkiky<$*e1oW3yngya8v^UGKH@ur8n7df{K!n6J_(BWrJ=eNhr8A2C@nm*n5abx2-!
zeZIN8Q@dM`Hg#dhh@fy~hV9(#D2=a0vH$&CDNA1NwQOL$K&L1XP-6~CBRRqUId=Ld
z$X^w?-2{}v*I2yM5)9~yO=afLN5G8*G5>RkEA4ZBSj9~Prh-)uIM=M+Z=>!8DnEf&
z(?Hp}93?!fbUXF*_JZl1?J@@WxK*dvB~cdO^S$v!01N7P7E?hzSO8M&in_CvWZL=b
zi(PK7X_xlrod=lO0F(G@pH8dMCA>E+=`NyAa<pAsjFBpsmv8HVxCsre@fH<t;SgzY
z*B`$23)lvV4_bOZC7zC@iQrAA!S%OZV;4+u;H*sGbEvzHTB017cL6=rI1Z(DqWIdt
zN>a9(7SLb>TTWO;cXJmmO_wX#lm^^f%wd0|e)O&uQ~Oy}&UogT+%T~8Z%vZXX`R>?
z*6Tz|p0AaRnfQ$CAD;!{b7g0qVG3_WS`RJQ(TFatN(}wXK-OkqV*{wIjPi((&|Fe!
zNk0gvkKE<m;;N%(q^?gFW#w~7quwJ{vyk4oM4{6D3*l^Ut{b>mv&qiCesB?*J-^p>
z7OyU8XG84(+bVAF0`P#S*5*rPWo60)D=j9}`2;nrB4)ydHO&{zcQmZ2`%-K@b2&3!
zEw@4@S50I-*mb*n%v0qnZkzq&`u1*pdSzYx_j2ARu&Fb|%(H^3Z#_IM1S1i2CVx7F
zpLldf5X`xDbIwf0wVw$$o~csW=NESYY(eY#`lK;2?dEcI8K_YmX;m!tQg?l`T!VyA
z4r3#su)RL<-QoR1oR%THgWG!*Yu!L)HlyKJZx-88s@VjLvYz!9t4}XzT7DG(oTSMr
zoaNJL;DFhyhg+P^&UUiYlN(Xs!kUfT<$&wefG&s9ULETcmo~m$fuWC%l`|D;zZ#xC
zE39*!00_ZUBJR@w04H<-SVYe`7Z{J~mx3c&RzMxe$%cfQpKX<*CMR(9)g+FF`IbLw
zj%u2}06tx@Dsihfw{IOJk3j%7)O|eV4557vRO+rQP6f(L2ff0O372--&DH+j$Y^*@
z<F=|cw|Fgx;ntvWywu>-IZjggeM#YP^8=sS8LX>sS4uHFaL*c)d!*!<O1tOh&@>T;
zPtK!b1mA6cNC!8ydeFGsT%GKeSZS_e>fs)VlW8j;+s|H~TpgG79%<2xbbH%YB0bLQ
zFiFc0Q$xA~3|KuFpgjq-pA~Nrp)iwv8DKPqy$3M?ml=6dK|#{2DEOjnMwCvc9!0mZ
zIk3n%fU3i%GCp0jir^78RXV8mR`l+G_0ZdEA;MMu)Jbn5_O$_O7zTiK10=-+>9bS4
z5hbx(cd)^}0o1|HHni60v(>ap?xUv>Ts5|1oldV><v=zmE*ss{YEJ&|=Dr;ST%&H-
zF~~yv;`#TVCzLiR8pQOD=;^+(>)B7~(o34JU%ytktBk&c{CO-q=j%pk5y`vZ3?Wk&
zLkOndrg;xw6_>L%;RcO{i<#X|-#;1Tddp7qgWB~iV+;T?Vn4397maxy2@Z>L_*_+{
zg#3G`5&z@TG;N!B`^gc1L$>MF%L5em0ZJ<1rw;%U@@jGnC-rSowO3NTX8DF*@+Sz6
z8I?ZhWPSgGy6|ZXr{br&jR^oTylE_BPvyc87?x7>{2N@Fx6iM*ZUEi_AtozO^f#lP
zybW8Y-z5^?dUa3a>N!<d(qXUAR6a;4b5Q|uF*(+C(r{9bAz4~+R93ME2|5NjKh*g;
znY9C4IvX#Sxx0ZX6moLL3KLSR8V4fY&>Kzg3P5?NfBjgADk3;0Qg<V<W<Ce1`|X;Q
zLLumQm^hI1w}X7PT$d1vqkX)WyZ$G&d}Bs#9thJI*%U4cKMZuq5{FeUQMxx?9p_=&
z2Cf?s#PN|sd|k-Xt+RhWHq58mVUs#j7PKUBb{_X0?jLni8n_3W1%Oz?TgPtBN{$GQ
z5~Zt1twMkyl?SMmg2=x-#6wc=0fqd|QyV_c>ZQ1HiS;cZ6!>7cgxaA<IyE+UW>n82
z>Wz&ZD7<d38Q<z>`RK-*)fBhBb%dq&)1sqM!!eYuv4{4^VyrAtj>SD9teNm|q5W2}
zv5<834Jzll{f%Ijw(j0#>u)Ho+O5~CD)4yFg%ZSb!m9RhF+J)rp$r>-IQ56m)cD+W
zZ#gR$K0F(xPX4U2^6mF_yVSLdF%B?;UcZh53%|!<gb%3-8=nE829oCaA>=m$yfH=g
zwEbyb{_A^*d>HI?Hd@)BO(eGVwMOb`L=b&eU6v%%`;^zmTD-$<J-mwgK$PkFo@iol
zPc$(5XT?y=iQ5PEAKB5_-CVW#9Khz}LH8;2Aeg_e!TF}NQ84v`2#JI!-*(h_aE$yD
z95dN<ymR)fl-H?{#+$3%ZXnWa*Ps^gk4<oNX2y)Yj46Q~$^J7d7h{qsK!*w$D7w@I
z2IP!A7VRA}K$i*>1Pi^N4$;Bh^+%(5nF;kW&PxV0Sx4U}7|XDCBWi$BjJ!Ra$%a#W
zUVLMv%wcl4u8EQcD^MWs0tjL`3{=p^?T}1{i4K@g0m1#dU%*TaYWMDrK6eR<r$<Fz
z1A|rEMkcy%1YiLx0JG-D%TCf=?}dqdRbz{Me@gK(i}d~Fb&^&`5*dRm!#DH`3#Fyu
zCHWr|6TPFphdh~U!QhAqyAR>-aL6+Ilsj9=Np=TfLn1{e8QTxpFvY$^`WG$u0>E~4
zoNr%wlqLzZ%Gn6F4%toEsZkl1Et0}LQ-0LM1Es6pNXT=;w<G}(SU#2VHOXxYLl1>-
zbQ{k{YSu`y-r|Q)D_@<L&3I0Y2*kW%{cIXo^O<51pbwR^nQf|#90n_GW?{5Co2Lh9
zH`Cf`Q^g|tc$a%&Qd{1KLBrGm!bL$^n0WmnZdGU;Tepc}OWypAXbKlFIg_lVq+c_Y
zAwV4tBx2?<`wt&8FjsX08}{7MFj?jND;ddsQXhflXj|7~fr}@qlFWJ+iAJu<Gh{B*
z-29V<qIX@mroXBlHYZ<y=l09_L|9C|Kr(46J9~)XeLt$L3x6xiycLXo)%swf%vl^q
z&V=;NA$E#-kI2kEPfE;MBlc}nv4};VRtoVM>-N5tJ<Sv3aQ96H-BmV*njiVZ+L-u<
z-M#={r*qgaI$?I}QP<v@37Yq9$~_?>^z7t|q@Xo`9YNuui>3%0w)-ELEY$=FYEhCG
z?l7dPfBbWQyW9H+Mi)+aTGRb+o<I(vJ4ntvjU#vq!OuJx&N>?>kj>hlNX5#8O`qZ@
z5%v(938ki|mZ-)9gb~$LIo6zZo>r!`s`TMytY3F=sy&hppkX3dB9SE9zg7Ie9csQ#
zk{%2bU82$<wrt?uL(jFhGFxhsOA}F4v&L*9WW%acpO77;B~AiXhXbDlyTx;;-Mouq
zLY}w_dO87M$Ftu&>oabvjJ4VqMJJKP-LH?9CRJOX#l`_2Gz8cL<oB4T0=8I`p0N=r
z4(m3c6JAhlb0Y(R9jDKuv3zWtY<s<m==yz_z=<o7H1F_6S(b|_Lp_|{Jet~tXVNR#
z?Cv?-lCfCow2AmB;!qfROP$>&<;rc+E{hY;6JSSWLM=kWS&dzPUJnMoVmjT=xO?ox
z{4sii<Wn<mmU<TsWjSO@cYD3A-r?b{*m{^GL(jRh!!Q7VOkMDV@V<3n@AmhzRt+wf
z7cz|v`Y@qr-k#3i#^-qD_;e1)c6#4zdUh1lOsGkXhjO-9o4K_bT$yJiAJfnfJcg}^
z3XsC?B6wQE7y{QmTZ(b&JX!Fpdl-6_9BFhc$uERn&$PVF!-%#Vtl3wkin-hTwk}#J
zSCM!!9J*G4l>P7?VUyL%+fGx8EC_zmuTmus?_Ab;>X#n!IM-wb!%Ke@C*S)MobB)D
zg++gNHX_W4*-uD+WVg-J^iQ~T21$&5?1Hs;Z*1%jflmH~+^Fns^LbW}B0ClJrHR&I
z<XZw<+Jj$AaU_fDte;}oOz>WKExh`3*h2<FUXUygGfW3bjQ2)m67Xi#FkNC2`-~E1
z*fn>osL#iG#=7iduw~cmW6<j*%J=c1Y2#$1Rz4McG=XTJB(+$Z3qdY(?nf){_ZD~@
zP|2IlP*+(x*!E2d{FEeYa>6Ff_~cc}3%|)SFIV?@tY)f(5GfD(E(c4gR_*4dR{SWe
zMbl!hq2%inM;eC>twyn}A$V?GU4IwjPvqqyD;^J<)uH0)Pxm8eSC7tolgEq*4s2!o
zjrX*LFK!4JfA=>zCS%N%9oI1!`tj8f<c*70L3-UxsZ0mQ4iT8AJs&Tyc*Qfmq#s83
zSUpRTzJn2)MEY>PbJz}K8UM7y4H{4RX?3ErJx3|^u97~P%rr+cMG$!?*-7pYtK7vQ
zuK*7ZnnNp)Z~^gLbNY$XZ4*rok*0NSZdW*Jp$hRlzK5B$nQTe0%86#b4vdmU(kdJ#
zW|J2$jYN6AToZXz`CS;D?~fm%sOK0?-5)jfP@O6zH!34M9|ejJhH5Q+%u695chOdL
zdDSmTt%ub^dS%z(eii|_{MJBkI#?Pey3lHC)I+<Z@1T!~dR+l5od*GC0grCavv^%(
z3qff~N4bG?*H}GL^eZ&u_u3?+F54!ptTHz(ky3b`S%xZk=}Y}K?6`GEP5h@x%N_7{
zMcZVg&BafzA6t<9c)MhW66G;67}9Kc!H>;{GkW-v_`}8`@2dNQ(Wk2ZxL7`{S0ZD9
z;7#sS@VY|l_C!ZjJ*9I|OsOb3Kj3_KhAH~lV!8x3jrWTJu6b@G>n4-WJeT>ek<Q0Z
z!i(RZh1x4M(fLlnNNjDTNMDh6RW|9ls_pDtq4sOCWO!xR;n*)?-}HxGWJu`Vy(24!
zqzu&X6Nc9X1-Hive?+A25PWW;(?a(=XUzzwGov#9PN`+0@Rnad-BK1iHFotFKZwC3
znA-1fVDmX?ag8d9U;276Yi?oH2>zBNGifc}b7?agEG}xtW**I?_U0`Z1q4JT9(0!w
zXr2D3ZHhRT%)+8Bim<-vvteY)TT&w-9uucGWV($UFFH!u$d(R=)C>LkS$TCyHlWr0
zs(-#KyUEupBQZJc-H#ek;Ac~0Fe0ZvA)YuMfKpnh?jd>wW~hWr;BVW*sTkXzg}W~Z
zz-e}~8kK1JEKsr>+d1>UWbfBkwjeA67J16jm@;}vZk=t}$<Viw9B{Zv-uGJ)U8YC#
zS3yZv(|s4hlP!4xgez2spQ#S#lMtocUO$cJ8KXy;?mi@Z$Gm8EJt@1LwMD+9f8{Ri
zYOL9K{kCsiJnWM#mXJn_fO<hodS?z>UGps_e@CV^nHy4P<T^b{MmM;JBvZ;EohuAi
zNYJ{9!DM(w%9S^ItBb2QEoZ0@YX`5ORHZgrx`{sChK-*)X}!pv;QcGgGN@Us`Q5><
z)V}dOOjrsJ4K12+U<zEzj&U}Yz2Qw>cjdoQs7Yq_wCX%o*4^FnH|y_ym+Vwycj5X-
z$d!-p)agF9V;~)ngh4vkh|yEN$|$BgWC@j&lvb7WC=jAZI@TthT3@@;v(+@&R04)f
zmeLM0W!h0iN5VbHD@dIWYebjKoEM*(JacDI^K|0L()(JOMI?8HROlzx7*&jj-UKnM
z7g8tp4-h)vN>AH0QxWb{pmx-9er3hU)tkujH;X(MhP9EAq@LdSp%y%bXvsB(l$snB
zho3k=Q%N~5)ZXF-e4jcHxf&-KBQf}3>N*>Ot8c<weik#I<T#$VM~__IF}EM-g|+f_
z1m&mV@yCYJF(B&tdp)rm++-#aM>2M4Y~h7y7d#D{1G+%D)4N&UYqm=Z<~7}=LxUV@
z`;wzF<Y)V3lg>vnXh1{58a}g-p~{s)M>(pgAwij^;~eIdvd$Lty@DRaIsS~DGZO+U
zM1?hyc~Wq`KkkFDbr+qpu-0`k020W8dMoF&1k9j!DCIz*R<=RwIKL>&Ej$diI1249
zO-wUG((6dbrA%qbro2Ke@V4cva25&HosT@ifPE&&;hpz`%$UBrJ*{T=S9Tv_abiR#
zr8u)p@NpM96{MffXe~q~ogsANUy7sAM8c3K4j+=i{wuvpyrI1{k$LyV9v|}*O!9sr
z5^l>CA87xaz0YT^$szPO_*BFD0RW0Qjyf;#d<J-tG6Dh0tsY{f&i_QN?~QRbMK>Qb
zDq`p%sn1P(O-+sH!5O4)2Ksgl;&B$J5RGR;wunmc2jq^~S;hfmq%RoKJd@5AeY~~<
z%l7&iYdZ?-RQ<k1dro+!twk8_)BA|_r`(f}Y2P`t9UFRy4;nYV=&vw7s4a1v@_3_Z
z*F?M?Tv$Y=woGqoa`u~$`%x%)P3`Hr8jsQv<&$9{Eu*ofm6Ok3d!-$SZG8sI2+4!`
zQ|A}lhexfH(xYZO4r1Xg*v`3-sJBCZ!B8fFDnN!re-*dQ58X?;p>)>5+Gz=2y~<(O
z7B2{W=`H%!_VT_w<W{nX9(_NJV~W2+h!-sGS9N#zIM*0Ekgvzwr}(t=#ft)vb7nvO
z{iL%qVZ(^H&Dkg(pvNJk$hq_Zuiy!?14M`jZjxJvR{*Ik$fgyh<dC>jO--PJPQheS
zaeN$Bmh`Cb)*>wbvFH{M4Ti`46O~?{=V1`4cUvqp1sh;}z@kdSM61`Y&*Py|PXO}@
z?^Y~uDSA|352<bAUhH3eBH;a1x74=MzuDd`|2MtP+^l~Q;@qF#+5e%8Cs?!pu-fxy
zlNSa6CKSV*^N*oM$sG>R;(wFeu>ZFQDA@l%h5+_>5ftKSEWz*}kB)gJ7WxvSl*J;J
z{>9rFdAriZ(U*WV#s{nH;J>^%($zNJM(zyI=AWvA_vNaAi**MPOX`{U?*D%ADfxeL
z1et%7jR!Ne-<-BfbpeP$|N1aXDnaVz40Xf}K3~g7;9>g@-Ht2)<gPSWGGZA+1R(G1
zG5GQ9PZ05cEkz2dKsMvyzZfOl1%NCeEEgg)Kw62tLY)4WdyBJSpq3GwY!&S|n7cl%
z{O?d}kTczZT?1!yx6=g}?E@H;Fjlfe27(dxxr$`PQGc4BC(~?ihTr@==uz!&kmP%A
zq;mm!b24T#mUJLny>h9kI316joTvfmS-mzj^z(`#=#mj(l%8ApOi}2GYzSPrJ2dKJ
z>XuVTKU;(n6hKgj4(xW8s{Z+8=)?}lvXHZxU4PvIz<l9fd`84hFuj}zF;N2;qbpDX
zCt^$;L?(436Ma%opn$oli`#!c!WF2MM{YT1oji(!M}<3b0d+ST*lLLfRr~2oXQ76t
zxp@V-6(N8@snY0e)174inr<Z-N~QrGr{!MZxqO(B@QG8+`VTG{*zZTWAM1}N^Jw1w
zx+VCNtz_k#9|hJVlDgb0Ip`A~tG)za-?4Js!-FWoI_azUw9CwE8VyK;BK={*+P8=<
zfOzR+Xn|N?FE3@BkDXFi#d~@>msV1Y5o$}U72}Dr0`8U*7X&puTco?uI|3y+5^)Br
zt+dd3O>2BIa_tmLRIKm-q>Mv1v_p`@PSs);V9uE-*(&M&1Xxk3Ujm5LBR#5D7q`WJ
zK+AxQqgjl(ckVcJ+h+)S&{zVU-7O;p7S?bCSh%}YlHCvX2+j(<FMxBcOO8xcOUJ<O
zJG$#xp*!gs@-X`PV(QQU*DT5W1A7;6?FkR#hP}&H)`2qh(mo~p+E!Ky$MW!xV%|d<
zkmO}wg)YDZ6DE`7L8ub!*i(!lWbe)%z?wBj8<c&CeNLy#zBe{(PVJ>A-r8;fDlo%_
zvtH*<<R;K`dm><%eXRgb14o+wx!s5}ASrNWL<t#?-_qO(le#pcjR7nxb#Z?Hzf)B{
zfMu-!c*_nHDNdpQ_`%(?eddEP`q*WqR$TtUm3p#SQ0w%U+E@cW(Dh`-%PY!DUC1iy
zxsS=wr@OqX<{U|O8b3rE&`to?N`O8HzQM3%38mmvb56VX!76q0==RUV#I03M0N}HV
zQ)Xq)`&nUdwyqvv&;|fRSAceW1d8PRN$;HRpPmAqAJ4IA{N&12i|Lqi=9xK+!3;FJ
zXcFnc($y*=Fm&k{6Cw1Sv$*xaGRECgJ|ec6uL;n+-GmWcNd1OL6AsxhukGeTduVBa
zUXXdp)QN$Y-TT>{-xWJuOh=fdUc2F$ZGHgQWvZk7E@pS-P=MOvMY@?F_zi-v`EYQ%
zPh>H28N<Wyh3CH)nKd3y+IzShWQ>4AI*4TcP%XoVU^-WIc&JHiV(_cEPY#}200-m*
z#?)6=0xZ0--!fKUmuBXWIXFBvEfC5=&<gl`j4&6SbUyxC7-bEFi-HIq<GMS!0j3PP
zYIIkzCjda!yqP?8s+tcikOxW2^%ORoZ`2fRZd$}h==4cmY)yt9AEjpPV<MlUp{QK5
zCBmA2-ouO>`LbqjG}U_}XBj}=yCtQ79&zn`)<Spvs$I=5JwpgswX<SES*e}|M=SlS
z2t`m1&L2gobB*#w)c~l2$fRj9fW%5RCRtJE&D7maE6P2zcITFANf8*?off}>udsOq
znY6YlODrLPH9<Yvk35U%sL6u9W1_JIrb1ybtJ0rA6>ejI1{7*Ni1J;}g4r0+PPfqo
zTzc=_cM14wKT2hNe+#!UgxsVm+N@{sk&a%vw}*upU)YEZQM>weKelx7lP~Qnj~HiP
z@;gu3P3mO9tR*ps(c}7#GFn&kBtewsBp6LZI{Nsazv?$c?3oRx=s^$vW;$cnz>I{&
zc6#G}46k!cQxO)mBp|vhi_e{@)$78sqqE!rZdI7;-{u174LQeOvl=Lfg>1s0nkjud
ziYIAVek*yrl|%p(HAo09y_0OOIkh*|cwOEtTUaNkI130fXZT^vM}~%ge6x`>SLJuL
z2`Susx#qW1z51!RjVf=_K=ddtXeLU-L2aj8FX-d`$3BTs<D@L+L36b(3yEPbKf){X
z(qW{);uex;Rn(#9LeIv+KU)`e&}*PXOwDPSy$ej*pNlf&VU$9Z{zNfH90(->Npp#{
z#3f2-uF&+`mYQ3Pk#!c2{qac{TgbZO%YWU*{|Hp&^;}RP-+3bNXu(13goyQLv(Os&
zK9*hzf|>gLowq<>%};Ix$U3CHPUx0n4_`Nj6>!f5An|1B<X42xaA2d>&}T}A`02&G
z=+J(|tXr?3?snp}&r3;Q=Sc+2tnGqP!X9aU$2sT1ou!1Em|80He&-j*iis}3rezGc
zBvVWJW$;I$$eX@lqvlzY9bRuUN||T+n<#;pgA?l=`GoF!R_sITL^tzGb@;#-CLI+Y
z4SpG~Dx`llf%4aR4D1dDYE|zDod=cKeg<y?mO!hScXzF1h`79&P)_$-u{#Cm*6e2|
zPABe~y`7mhQ}m48`6KVO^m=Z>AvfnoE{+ehmxbr?ltZa1En7@40Mhp-*=isGS4SRT
z#O;Io)sIJm<Z9=*04y#>j(D)xDP9E_X2(Py%&MGxgOP6kHrtq*yY{RG6iNGwg5)tk
zaxtpjso8wvA7*sWb7h50V#7;@b&^HeGL(VmRwU32j)1X{n!cAITGGL(K%baoLH@CR
zW(7OpfbdCq4hoOec!v9YIXu!6A?wx;S?<wZ9$xmJdXx8a<5XEU`X>$E$JHxZ6P)lJ
zb%`Ub@={4#Vep@}cIhbNb{e;y+%)WuqH2&6G4bD;Pdq(hHnaA5Ai*n<6363KKzYgy
z{MUw#&KQYgaTg>c^>IFgWt8ZMt7h$+M}whP?2C%qZoNj|l8fCS*|HGx-o*C0*Z^SJ
z=bn((8aC42KO{sPx5`SH6D~05woTI7?daO^_N}Ce_0qaMZ_F2#9+r4NfD|~%Ii?|+
zo`OrJmOG}Fdrl;lkyi-J{i^%Q7N|<TER%@Qa>{=m>jCLyLy*4;vMJD7qq2XZgJY5{
z<*kfy5y!NQ7)NVEJVtAIAi|iYm{|8)NGhT{>I-fV4ZwL4u=J~905@b(C>BX@uX#I3
zGb@LuSw}^5%Y8@m7L|`)_EC4B|FDLlcF78l*`guvEc0|@#+4e6%L}y^WE@N%yO;=H
z6-i?Z(h2WM3n}D;%Ga1?I+$wdS<y_Y=>2h96q7kM{`|x>I<awjS=wBRntUSQ=4zAm
zji${eMoWsrXB23${-#<2JoYiw#PP^w7RM=qXNL+%$6#=oTc4;+JtFcRtz${sodMHl
zAojUl%43D9BlYZH>ZrZoeK-;KZmB<|P3AjcgcrHl{aw)Od=wE9ywUdSQbwnFbLDO4
zU9|YGYPw!ri=|%{zR3}Hr)HpwQ3-mCGQB?SHb>dS_dNRXY8?U3^HEVqtTXUjwQ*-(
zxV1AtF&1Z`Y46RS_DKy&ZW1da7#FvJA~;pLV9AT;c?pcT*JF`E3+wFAc<r5Ut6-RL
zfCPg^AA^h}&O`Px+l}O6k3td;#u9K+i+z26G&aJg(=V#0W-dyc+7LT3LG)J}(SuQ<
zMKx$bREA(HWQiMzMWJY)>yx0DNaOii2N$OBx!3DkU3II|;8I3>#j+QV%R7Eun&Y3u
z1;Q6(a#OzUUw<@$SGZ~oTO4#og<BKH*`Q;zSO`PA6&9C9<waT@HgSE$@S{h{6+cAF
z^bIp4JY3a2Clgu2D_~A0E;ZThvlhL^7Eyv5B6?Ci!g40G9sI_TxcineY2#k6geK$|
zd3^^8JNcJ6pFftP2JODDf<y@4LC7apg;4L8bQoqQ0E@w5ll7~w17imYPzE4{3=)cT
ze(uKo9M^St8pOneyVMJ&O>%sWD~?y8EHz9hp^tl(^uom&13o}vmTjUMhGj)d1Wj4%
z)%i8l7Cks|k5q`^@KNzwWB;YZFiN}P;=PAV@$d{;czCccyGksxiDYs-;VIx9{)y9L
zgA2kX+0SraN{~N23elZXV1m$uMpDuq-uq(boZs@G9$|CL&s{ex$7V-lea@N^vP~?$
zRg`31H?6hRi%(@Dgrm?1y_~3-xh@K<j>$iuXKWC1^>de^7g37qor}M>6dK+3%M8m*
zjy_iQs3=I1Q4!?GBm)l*$}8KeC_!NiS)F)=zRBP`d{UQp+QJg!s(3iHUo4b(>fxfz
z_*7wh2kb<;gRufaqvW9Hyrbv0R(^W1lKlMK;x2gi@Ov1l!wdTA2xYL?u=nuJ0KahH
zsbLTeh&Gx;?98~*ntrC`z{<$aOo|!uB=|ysBCJ>-ddKag_(Idf+h7kC_cLddQhjgX
z-8UMu<j*Ih%^B(|LB)7z2O<jMh5Cm=0<7qg2<(q7!TW;#q`O<MdVv(#D9IlApsH}3
zX3avC41&sV@;y`zn6_zas)vJB_#Y3_yv_Wb-elHJm9<MmHFw%3ym4zRsZ;#r?#kt>
zFgAG?yggc2h0W)wQ6stJ*{xn_izv;<46C+0X)|PP9hbIkgFt6TLl_W-*!%d+5#J(1
z!uRfrbWWwTUK_l0+!GQB#zhR>qUK}#k#f!fHJ{*k(^1s!@R%n;3KgMRryt`Pj4ub3
zB^jR&p-0u#oPSlvftA`GK*zTnxgL9ZGsz#0>QX;M%kI((!jWKU<irQ^C|p$AgQYLV
zh}Cg|@^#*8j3O#5xEKhIcvhzDje{zvV;^*sIlPZ{IQY(lZmv*zM2ir#YokYE^?xCP
zdqP_#D=2k<qSDh`d~AVr;rn48xBG`6g5>*mWF(vIl@Eu0P{yQ<I0u!R9q8=?@JRH#
zAFgU}pyDXQ(6kIQY&w#1f%_wNx^xA+eF`2|NA#+#heEt6hkr-e?V-L!7(LE4qZ*#L
zvmrQ%n2$;5qatb%sxA=-_VFfVz5^4&UBmU+Dpq9az|nV`Q%DNJlF9S85A<oda_FN}
z*@UBZeEf)9AL6H?;7sZw_t0Anf0`WPotR0|BcX54jJ^8lkV{`A-HV{5kjQX0^}1!`
zk2|mmq_QwELiyN=iRca<nu9r16#&*@B6z&wOemMNWnVn>*1dPwDogTQ2%Aja(ILEU
z7u4S5Ao~vgw&<GZNBWwTFneCLQCy-BR@i8Cd7UzH#sZ-b#tN(OPs0CWOmNJ@HjPbm
zB|Y5B642|hNv`+;%y18#FOeQnH?>dn{)CFAVvS}=bauc6j)4}3euswWm$#_P6?;L-
z22cm0$E*>g>e$>W;87MK%LogSVpiDQ)Y7vLkkUUY*dX;;U23}GS5KWwX$F?QI7u3w
z!i;EWV&UWTFbRhjLQKKE5vD={LKhpljRthWCG8kv0v)-xDBO|Bq2))@P#C@8sl<cw
zeW-7tT7V<fmW5^s!u<s@Tu;Nq&QfY)@#>@cw58Y5gw5<Tla29TlgE^g@zD9sL`++)
zlBSpI&t?Tpdfb2YtGmJdV5W}0%sUWedq$H_5QNF1^G7Ah#4)1F)<l6oVefT5c(2;^
zow+juO*3K79gw)2^w#=XMvMvZr=2?%W{ABq+@|pM&ANsy^Y`H)L7WrtRlQWaNY9CP
zUMT5Suf{*~{*fgee9a&?(^sARq%sypYw2_=)NxB7nQ!?F1#{>$*_5-k$L`@pBu5OV
zZN%(Eh75*X45vFRt=W{byo?>CLV;fgx}jBVcelEJoUgxSx|=FerEeI>%$O~Kuz}I!
zf)PU}Z-wX^9ITxvAq+xtRL1%I?(@G#b?Ph~W{Ulv*r(EByd(1}vv0k9Hx1`<mwT1f
zx$+w)J@NR725J7>O#o9f%~N$v9K!4B=2pB~Ul{V2ul8FJy2iSxCi#T{hF<>{7W`pY
zi}OVb{x0uKW&6T09OXu~kp*Qcy=+yaKb3;uiC_$5j3WPq&(@JTB*pzzu~R82mcXQp
z6?RNHf6Q-GM>wxv_6K%2p5Kli+65g3aWj@5QGengffuVLnc&gZU%vbt)j`jY2^t8U
z6)L8$mtkw=VtNWM^fs~(#?dg|Z~m$(14>Tm?PWT(ZCUws-GTf)*IGonR|9PYElrFA
zH0Y>;Vaj%Sx&BASb@m)<c8V&{rh|<7LZ0AYB&0))JcLe9R}A!rR)3vyaE$-)EIIJZ
z968Rs$dit$Lv1~9^T0@kNaL`(SLG8m`!o-rHw)iI<b=S3Ee{;RWBbSSN-Us5B=jdS
z;Qa^eF+VTlB)t!)l8EO;h1h6tNJwyTPQF*aOR2bE;%?5}Mc2~xDlgqeSO|j@<dRfr
zgO{a~dSa~C)JQC0;R3DT!C-nE2F10q9Lca<Xt5?8iJfH|MgN`ZA>HN3w2NRu=!@P)
z=zcfDr|9!dZboNSICKpdNk1rJb#HtFms-4)%3fXFJjxA%4z7uj`0|ZExhRH&yy7|5
zMEm?LQftklXS@Ci=ECEqL6dA10xMl#cDE>&Cd|STXVZgI*DS9H2A#koq)21j*RW|7
zhdeGzgCt+@teMwAe9JH2qlW{h0-dGRR7;f7j&ZAd{_PBUsVm&#uy58xwnOnOK9LRU
z85eq#C&l(?@%(X)bw@%4_yj^?#J(5(P(gG_zX5RXNMTr=AN5rdIEZlKTi8nm<NZ@T
zgC}+wGJiPY(3@qd`e8{XDruaaCRE{RUpERs_{rWR?6Y{t8Ec6a>`7Th0s<65o`Ssy
zA$$Zp(f+BsYR)i@^M!EENt3ehw26q4UjO2(-^i>1kFjA+slq(M_haHjjPS=O6)1kW
z^cXZhiia{QTh^aBK3XW9g2~bQ@+n>GXZg9z^^?8xoEj|Zy*N=C9PrFrw>73z5;#0(
zSWy@elnicV-~{iW*Mh`g7s=@~EmT`;4S3?hg{6%y__wVsgu{}~z~MZ*3kE&p%QQ96
zuZs=#m4sXAyE3@Y14Jd9IsxPsd6jvmvmelhJEPCccI2eIUt~ba?27rb7PHvOR|_Sp
z*&^LYs@JGve)#-8fA{sbSUL^k&xTWdp)n>%kH$fZ*P>+Iio>CP0NC$=On-{SR?u@p
z#qnm3lk5x90rxpdpR<}|><zC*+19Yqkm{btZF5lia-yz|?0r)U6Y~me^@qVVt<o>+
z-FXvf&S{^FJc|CXmMAtp0}Y$$W$=tn`)YAR5$0*&!!cr3&_4%NOhx=i8|;Z+F=<hA
zfPWNMQlv)f;r}EXwcl<vum`nri+Kn`CM~`IO+0s}(|`wx{hg!av36RrotW)@-0aJt
z_nx!X*T>Bx`C03vYh1f~GHck9UaL84YR=5M6rqaI%w3h6PRA+}9poO2butiWYI4ld
z)G|{*X1U{BYV?4$v&D*=;qLtVA2ZmF=<&Lh+GM!Rm#`R{gv6N25@nok1v_mndfyyZ
zkp=C&b3H>tb#BIUlPAVr@CL=}<AuRg)%2f`10QF*ugIMhnpW?@nQ7ObFk*pS#FMd^
zX{QzY06G)fXysazOF)?4qE963FsD|O#Loeh@Py7PU)(=Ieq1qMe6XyOX_o|aBVDB5
z2(It_KD#x0y#shCx5cJvp7$hIaG90vO!l{D7y9i{eC7g7*Sk#;RQ47?3=EeJYFd<7
zkf#3l-l3WKPY7BBS(#g(&f0)S&8Y5l*#ep^J6=gvitH&G-CIJF2#?_>lf*nL<czVe
zHNx;uL1q|8-i)O7AeZ3&%iExI=DyVTspI&e!^c1mpqvB!_NLIcy~&CpjMKaDzmT5s
z|I%UmFHzxtZM{7wfL#MUx0Q8LAHF$8{9AUIlS`vV-T2?6S?xw3pV)(rDobXIGaiph
zo;xqkug*TXxz@dw)5O@$fqQ@`0$--P3=?tk9t^VDG2VT0u<Nq=B;?GqHe>o?SSpur
z(o^|g;>IE8AtkzQf}J?w%(JnSqioDL<KfEOL4j*$)XB%jtD{-h-THrv9}OP>8)epY
zmR2w{IBxairsa=w&GxlxyDTa(4Czw}KHZ|ucxd9kOgD1+pHsreAOZE@lWDhQK&{w>
zv!SNv0%~sxm+1-{UQ3>yTnJu!X$7`mQ~&FBn9o^b=X4={<-4&qA1<$U0}9io*8hF6
zRya=dIe?7+s~<4m4gIfRAoqXNX4Z5&V1I|FA#gfgK^3^hA$1+VJeG5^kW_!dy2vp9
z3k7Kr+eGr@#mFCe7z&O(Mf~9^9?W)rrzeap&9`J4@Zfp`CPDw+PvjH&USjax^O@w(
zkEPk0>se`B0P~x}!ja&0n3uZ4SP|4LfjO7(<=dMPHjLq$<vTaL{@yM7UEQGDzN^LU
zpgbQVv9M67$R<o*Gl18lU%<uZ)0B~9S9c#jCgb|oIe3hg&Tn>gQ*Vy}WK4KdQ+syx
zBPkoej9$X_nsCJz{yE{_Dp?!>Av)6i-k1_)F&g7cc3BG!V8#4a;0FWFY2|elzOoMf
zop?U;W#DdA(J2&X|MsJED+3K1iVEy#jya%k)B+$k;XU`Q7GkOVw2^JMEHRG{X4<<Q
zAO2-~_frO%p1^!vu{N;QY$Jd<ELX20shO?3gZj?SzRcZRpDbe}C%`*yh%u$+n#c}@
zmS8%?b*Uq(w=1SbZ&Ac~D?TEK1kV8s;;;Ff>>;Kx-1Czp8>ZLquPD(6KIn;ltS)>4
zpg>~`t@6&O_-*#`9OJ#g2w0x3oB<=+x+CbBE7x$o!TLHhU{0-!$0a!{AG5p0D3J)|
z^9QhFds4vLt1zSKJ@pJ%aqsJP<y?ai!1az2*#YBfEgHaf<)ugVlvW-5G5iacQa1-u
ze;bhj;HQ4KgY4*72O>^;?-YZMG4wXK8{oBiD?r!fxnZ;_@yXZWPE)|fZ$AM5`VB0*
zMZiPK4m5y!T{*UU)(5@zQeWeQINKp+8%C+UyXOoiznYpZonh>(lz_pb8xV3#Fp!dO
zJv06jMgvQDbu2%SrLCj?{Uvz6)x%#Euv%IH&wf_}f?EUR+2zO1pY*NS0SB;ZJ%cRG
zV}GEFGO?32Io}uX?Q_pv%%1#h5>MD^({=*-Yo7qZk=qx~NxRJ`kZT7+*l@`T^F_`k
zFrwQpobM0*)5Y5@g)tP%lN*<T%>O!@AFbmrodL}YPcW)LH$Z80$F6~;Qu9~#sGRfn
z5#KcJ1e6!ZrpcR}?@y&00aa>K65UFsURHYicO-AuU^<F-eOJX3eD`F8yU!xwJb1)>
zh0bj28})!j^^}*#&V5+DQ%*niWTM;}s+f6;anZ(xkwZ2WV|Zl&Prg?3E}h+b`HcEp
zfRM&<o$c@;dSENDW|(XVJozdhQ*hJvhdon);AZG)%nJ=khxA>Ek>y(KzCWSkz_bFI
z+?%ZcSMAT>7{x;X!*c>~MF-$(7By&kzf^CfWk>&=pDAS+kNW1tFLXSxxGVdNrJ@di
zxuxyg1T05GiAG0_IiS5kPsi9}gv*BmXl}Oxa?p)8UKm6B@>TmYPl$8<(FWsF?k@q?
zXWanK92J~>JcU7@KL+NL=Ps>9Z!QluPxm;U)V2_byRY|u#B|X50<4@nE6v}#_of-2
z`y=q;fYR9*j5X~_HNUA|S$&-;32b-+gu$bl#66;^3zuw<QT?o!JWV**BF{60Pbw;u
zO4Wr!7S)Z9_euJ?E8M*tUuFI@RFBRyW61#cwFvhZFmX+k*B@%W)Tr-|bi^W!Wl1zD
zp@IH+Z`DR=lfMq)2bx5BAL$wmSk#1*0X{m-U5u0a94<2fSX**>Fd|tOdlrY(`9RE@
zSsu_;qOs=$--wo`w)1ooE}fBw7|?LhjgbQZZNvAw_hkX#BsD0ErwMu%VR;R_fTG~&
zHsd$Hb0c7l3hrWFaZn!!@5Q)ZalhI<OH<wRJD$jTHfLb}Q`_;8qeI1nb-BM-t~#Y7
zMiwn6oHbYnZ5Ngez8GmZ?_KflFrHH-i3nFbR*Jvu$dhk90_IvFNB!ZT72y2Nw*wk}
ztR!gD+y|A9%EX?Ku>rQWNW7i0AvH+VF~)AjQQE`jp+;eEv4CY!3|s%6xY2x2ns${R
ztq#W~`xZ`5Rf~DSL*6u%eZEFFshk_25O5eemTRlJ1=+R_g>*CmEBzBj63g2mITj!$
zk~r)?Pjw96-{DG;sjuB_@PPq>aBV>`i@M8raF$tN{qe?&ljH=<Qr=7%R=1cre3<gK
z`JTt@W`<;?a&2`H0Mh;E4dB%m<%NFR_~}e5BY=;6v?Bp<Kn$L+95(`rSE-obWKjXS
z%99P43sQjkZWYPfmYwdc+Jr5lvZVqT=|Su&9K~|WILPPs9}!4zWX?YWccIrx{aPB{
z;oDhYTw7)g$LYVxA$2?JICP(FkgyG8SXt?pM!AdQ2SlkP7ui`CmKeEhp8nX5rrF<e
z_lP+jbgFP@rq}%h=v<Bg-c(Rbr5gqtz(_6LWOC!b|F&TK6dA{uHZg_jQ=tFKOYp%0
z6RomLCKK&Cvow;_TJ9|%W9DS8q=$i?lBtQ99c|3KZ6QpcLH`onXAl9kFj-)_CH(Rw
zfTcL9xVj--68nf(mNm6A;*$QQ`!(jB4RkYUK&Dad=Vyp%lt_E2ch1-h{#d(JqQbHO
z>K2fga~A=Wlc#nIu<v2YoQ-6uMOa6Si*=!D@k%VBK;z3BYP<vB%@#Tyvj8yOv^d+N
zqVpKm(6*`I(Q!zJR8G699DQJnI`>3(vW?Y@2VEU$fW{)JC8jLIc-&=>Uceg_xQ!N5
z<*`(aEE61Q{lSwZMI^Y7LDPF2ds>D#<BycJ`e>?vHZS{e9GRD~>Q@?#)uw%7FFTz5
zJGsHlpUZ#Kkn6ad&WGvN)AINOyT+)u7^wyO%ioMkr4_xjVI?GPUGuM@uM7`I%#|xw
zl+)Tx(l}GjH>`Kn%G&*R#1F>bH#KjVyufVN&I{}8gYEOpq$HSkk8Dtl4aK*T0v&0~
zKnJo~rCJh+PxaA+zXSK7WFlMCJP&E88F!0uY$G6zR-o~HZar{HTexho+PXsfbgh?W
zMMK~Dz0#-Iq@Fuf_yU*?>}Mu_Q8r7<iA*nq^Mv8=c>KDBt0g{3z1{_b>&}y(Chm=z
zaRL#=^Ml#p+~taJ`H&K6iT!~hK?)<r_o4eeh-PnE)a6dKXWW}Moc$gKf<e^|dK8*b
zxUK^J<NbGRzD}%;dl{$se%PkX`>z;tS)YmmBBcGrxdNMTraKu)wZN_9Ea@fk>La|F
z99O_MUN9?nr76W2jU^oZBo+E$<|S*5^tTvl{An!sPjrEgnF|mq@|KOpkdh)Uxru&8
z2fK>KZj%JbVfEu3&5M>QieW(*JLoZ6U9CflzuNA(SkeAA5nzs7!7h|qh^ZSKQVowy
zDw<@LtkqcW`*i*z=6?|Po<UJ{Ya6Bt5?(SWL6QVPC5sXpkRYIfpa@77kR&un?j|Z3
z$&yo}sDMh4oI!G?$*BpFB{ZSQp<&kcJ?ERLnyH$qZ+^_5Q*ZUz)iyh=_1w>WUymL0
zUI4K)?BprsL!}y0oL%Hy*VoJLjOH`@ja>xmBFv4R02-sq%F@g)B||fkCBe<u^+J>X
zrP=QIO{JOL{-8d3yyA3EURIKZ!be0rE3Jp;%7FapW0T`&>A{WW2oev2U(kYjTr}2&
z$dD|O&<?m5RGiJr)W0xZ2d5~T+2x+b6ithh^0$TZW)m_k5iV>~$opVp3AdZxHG0~y
zm)B$1*Z0!ysx#-+;5vs_l(#Ozvd7D+B>72~`+*N|C_YGIoMF7#=R3tB&9SKfmkd(u
zZJh9!@#{I(MN=22ISy|5Ecnuu2gcN|=}3@@`DsBVS65Vv<edpP^5MAf>o^JNbPk?R
zqE3uNF^R-6ZT%)v9#<5ixkqDEE&O07KENfS-Dvv^hxo1$5-^zF@LQceh^DY*;=vro
z*wBWAFRFAfrH?$@YP95C8pl>mKhIB>?E6wA1SEK~m-t~S@t8g;q4EdqL<PJ9+*1*k
z>UKWtm~q+j_zXQ*b_?Hw_Cm?LSM~NwE?&Iynrk;8X~AAoObLew-b<CGyyN0dD>l9K
zmY4i#FCE3=u9l=Lowz8om`wRnES>Yha)afX+d930$88R;FSkPWBc(SE18w26mqQNA
zsP{jW2i|V3`I%G6kiz-AnIpS8@!=$KEAcma^B%4K@I^|;b3|3ckCzQ1^}z8{Sp>Sr
z<b9*ip7D%RVN8Mnk;ZkmNF2IOY{Ak>5~l8r<7P9q|HztUaPD97^b~y+P1ivsLakv-
zvwL_jE04;<qh5RZ&0LvYo(8=eKN#8Y@3?=SX4F1S$Bm=0x-?gX`7OTiqHDg?*pRQf
zwC8QM@Q7rvadC;N@^c2h<)q2ykZz~?3O^fY*-(<gU0DHEQ40RXE5=p}VQtTH{Ir<U
z(+p#TMu!=)#ZgiVLl^FK6c@olV8Lv$s#<bR3n?Cp_xcwpK69Z-K!ih~N(v#4%w35b
z>mz1zFWS+|z0#gdiF$6zAm29}sHS&+Qq^88Rx;nAu!VlFx`k)8-@zCzPV`}#XO2dC
zif!--SVY$eyD+&zaE%5iJGLCm>~A$iM`%vNYeUQ;GO??CjVqw(^UZpDQ)B(KW;y9A
z)H|di_I0XreVms3xYTUeFkE4UNWl%GG_EkGIS?z6gjlO&LdkcVG-Dq7b1Yr)!<?2E
z)DSMkMSsZJ@+x0*xlnY!)o=U?bJ)+AERwEmTi%-Ae>X+i>y>F&YGsgQ;94-X(U|pn
zy;fg`?@OK(xqEB{W+B=h8;X$PCcT0m=m%8pE1|}T6LD9Dj5XFDq@~EC9IsiVdY0E#
zJVd27A3_wUn$Y<{<2fNKtsRRN(;Hm(kZwlAG{x81wk>n$7^Bimn796*c=d&)F0EVM
zm*p~!XV90T{n#|1o*$wCZ-t4=pjC+-?nDX6lr;HANL0KF@Jmvo@mwdoYRcSseVxZR
z7G~R|B(8Un$$}t&;i;8%Y&{(=myJ6;7-lS{bRm8E3r7$oBasXx0ZFzO;D8Wd-+mG>
zer^01S-SF++NqVu<*s}U<4eCkwy`jOpUPx>QPXegKTq17Q^SL<d%*g4Ee*2u#ot6P
zAG(sF%x3R#G(O)YU2&(|;o%zmB8^**o^1)X^kLf&L-J7S!I~)9=vg$RyE70mjWuuH
zkVg59InG{b9i5!N@fNUHC{-wxMK+F`-IVZ4pB2sG#4}2Iq~XS?GppuY&QE=ZECl;o
zT<40DBm)ejj9NMkxQElBJxaz`*4Lx4;nq02+ExQZ#=Om!(sbvpSj&^(jPbqaW7DP@
z*#(*phzajhEa>^bkNPi9HcroV{P1pfpX1fRO79C*P0UE>Nm!|-!se-5TB5eNPm+bH
zrKqds7-mBTEd!+Hh<i5uZ-YyttkC>(qGH}k&oDcLf{jT;p89euhtE-Cv%;%rZh{bg
zB`Gt0)`mFyU!b2!{#8ck67DQnjVC&Bi!q(8y{amjp8F9q5a{RKJ=FQLMYs)QJ$@vP
z()T;PpO3%G5vWg1m#2lZbK3s?nwd!{#D9SihSK!l!b4d+?_9t8t|};byfLLSm(=35
z)1Q*KJh@{I?p9MJk7jpb<cJ(Gx#J(oOK=AyWO7|^c{p#wGR~O1#9?1|ah+&bWuC=P
zO{`1usd$sv#DZR0<r}7>#V|UA`+{CZ%_|#0CSqba$D^kv2q*lr&~3+S?!P}riT(1T
z(kO_P%yc+w=QCneI%&-@nr2bP^-!3vTr*nGv;Gvgn)!Ir*+s-5RD@p80`>Z_PcY3n
zKO_}cJdXhz15)Fc=ePxohn-QYXNm@0%+r&)wSAV^knI4sHyzIF@g!1K6jo%skqo`_
zf%2YYkpaA81`>tq1|-7rD9q-fs^SJ#@7T{bU|H|WI~l=gzG5p+PBlc1cT_h1?*(z2
zvJ^6*9=TwFh?YrUII*Ms)^-0RU4c7(?g@V*6Zx0k^1;&C;>Z+!-1WMN*sS@wR88FB
zhE%)IA)AeS4f1MC>P;2*FB!%YAs#DH?KWY8ovDm8T>NrT@YA|Ghxv77d2vt68=f(E
zHghPZR2dRyd<`_hzyo@Q-%XYm6jv7Bzk?rE`RG|Kh<_f;T$|_2%gBhdT;A|m5(t2^
zDo?&#b<Bjf@)>x|F-W8g@T&mnzK@G<bfPf}w^P!!d4I$%`&LMi&r$q8&Yy14v%&+3
zo9D%MLXA?bgK`)j*SF5;%B2Wx+SFI$T0iTddhK1R?vnZ%BvRwcUmR{m)<&p9L6!EK
z-tS;^o}p6fH`?F0bMWq~+Kk|rPyq*T;_l8k#EAR6f5I-)Pu2Ehp<JlXh(x(q<_QN!
zI7jEK<^BWjR$k6}nc${;fqnYP+#_19r?M={#~;1cWw8I=@k9~(yK_t}<X4>5UHpG>
z5{KNs8Fqo>+nWyq<8B^*26lh8E?Y`>5{}y^d}davw5$!Mx0xDd_SSM2E%{VkgRQFm
zK|H}hJ21rVu{+ZGYap{8Zs3wAXE!lb;pzv%_6=9V-U>7D&Np~Z#{P9&uRb*4Kyy8U
z8SOux8Pt*6Sw6Qy&~}85&Ab9@6#+rZsa;Cjl_%oZDo)mS8-g<Z;v%^q*k195bnjp5
zxXbO&Qb?ZTS(|#3TQo)EQ*`5J*FVFMMDO?Lbsry1572p2HNF3rKk$V-5(ueC?ha?O
zF941Kt+a7<+=~7%PhAz(soEE)JEw$9`)5yhPenVNGiQFyeUS4{fPUnF?km*}QA8n|
z4c(K8)5|$=vs}h^uQXy#P2cV_mI4QKztBtB5cMEK!@rJj<&s0kOXuO8LJH)hR!>KS
z46gTuRP?=T-U`gn+nc|LEnES37CQW;CI5`#!iuB?Y-k3gn>KF+oOi#c9)|?In`nXv
zr8c==kI3Z?hoj}0nt;sc<8NsA*E*i~QsY{R*8`(a8vp$lyF^Ej%y)HyPWG?7Nb_$0
zERONs%XRQ6bTsrK>-GOJZvU?|;s4WA{r|ve{O`Z<e*-xF@Bfd?nOLhwUZ6T!1Ozz&
zcArDjw#6VZMuijLw*HSI9YYM-48CAzya)uhzgmXmYR+d!cQS!shTjdPH-84~|EZnv
zInkb{c>najrvgumPk?Ft2DWY?P<t9D19#?M0jw36j*EaOlmI{mnFg2gdQq!b0f7O`
z=p~@HIZA>YWhWt>nkEX!EE8@TM;L7L0Sf2-etPwfKwj+be|}W+IYIBxW_$5g2$Xsb
z7n`uQNQIYu`==i0yxI+$-kY8lV7v2!&Dyns?Y(jxPoWzgIN@)CB)4)Py5FsM9XLdd
z3*1#YW0pZ8|D&4N_^aWh&JH05#+$1+y_EV1Cp<u-{Wv~ma}7F^bwALpY;R(NOu55y
zL_Eh!o*Ld{tY2=;Z2U{M3{{#7gS?8f24<l*I-#&-Jc-680Db5{LgpP%(<cH{<EK`6
z2%2aC#A6}wtL<)-Lvij(kUr;|{c>vHP^&FA<F%6_28n|}P_5$IH3vpFKIx4Lb3Guu
zbwWZ9z22*s{9f@rQXD_}Z?+y3hY1gPlE0h+r&sP?*Et9r*FXqL3&2Pe0%X5e!v{dT
z{B~^Z39%yp0mO&m!P9IpH1Of-i+kR5ZzSa%+9a}`4P?k)+%tuE3!tciCqwRRD3Xry
zIbIPQr~#7W&@s?-+;UvQXh!Lo?ZL^thn))(>P^D_=Zd`hI1^(<fg9rdZWd0{V3|@B
zvIdDjvOpEP1I&r>P*cDH_;B;TK!*2RZn_4gd{CcvLHG(CjeSUF<v}6lihxL~%_-Z(
zUp%B44%K(y5CMy&WGg}Nks3n-#q|=%lY2CfSoa)!igE&)a6FJbi~m{#ftSAk+kJOX
z?OyW&v;p+40dR!11IUPXjk%B&fIhxQP8b6r?|c3Nphe^x?`PoGt@fmicv7r0ao_}x
zyorHi%+en4Gx9#HgKR|QJ}Xk?djTlOQZ}n+@b*A6^nT3>tgx^1oFSM2_bCuwSiUr<
z{oB`IyxI?LOJzvaxQ2qbkJ@t+z5cs6F|KGxDerp3m-n3XR%V{P1saHf*S^2C<|=Wh
zJF*OP673dN#3p$}uSG*8ebzRQxbzcZl)(a1A%*0hkN%wi`l$;UP9N3KyiyQ)WsE2P
z22-d5?$dY^{HpFfDx5$%YxEJM0@0cuL;in@x}&8ZUnavos7m4ZuMm*`c`Q`>yaNXa
zltaViK$p*l1l}As2}nkf8puVVh;Tsmy&VO^sUpGS{<lp6Fi0N1WuO*Blwk>UfuBW}
zGhTdf${C@c%2mMc7b?t!KH04IO$^qzzT{9+L?w6n=4ijX-l&Nm5pD{c^50fg<?qti
zF3<-&Rg=T--m|P!M+L&6K*8|X?i(ybqCwfRnR#5F$saTYVQ?U2xr9qm?_Fjv{As*1
z@R@-FnPXH9fC{&fOE+{9-1TP0UpG;R=W_5i3W)(2o=1h|2BdM^z(O)S7V}(7U0aUp
z95iv9)NR=bC*J&u=KZ%<X7*g(VbgC<8b{uqT`>zzWVuhaS~G0tW*BnmqOxEMtpkW-
z3}g5*5SCx!RCJ_k3L5VqULy4fR<-F#sc`GO0NRIUFmidy1RmQTXLYtw?hGAPny#!s
z**7w_oMnL&)T)}gv;3cp|5)_g8_iWRF)TNs+*aU@Dv_T6D%I_$`(2!7RrBt7i@y8g
zR<>iLIH8hecT<kWdzlWZHT4s6AbDVWlQhc3!9VBhKhM=uQn>8Uk~01S%mheef3-o!
zS{zr4&oZP%78Evga(y&zX9i7=*ONaP{vguo_wpR#fm~NktHaIm@PXJ@t{mf{!3JO*
z5e^6clVl~*T+>)H0=R4|Afuw+kq7d=D`@{ypit(~+VQe9CuSc4sBPYYj;tv=NoC@N
zhjIP9-U-1z;OS9@Yt;fh?#Z4e@zF><6th_6w`=N(*_B5jCyA9&|G8&q&mYutqu;Xp
zBSW2kJ!t&#TjR9(+N)*M>B+YFm8(K<l9p&)ub*3_N?KLEMVvm2D+z*GdZ)r>6Gmf1
zjpMAyC8Q2p|JSgYv?ZqHmEnZLMG}mrG6u@8z;!Dw3l*B|zkKek8+0&v@m(Uz>|W-p
zlG$ph0INUD%v(-er+|@YuI>v#qd<2?Pf?}WZE)(Mzpi1r3xt09`<V7i9t$9e2<e{I
z0fBZ^V~FEJzLwzG`+G~<!w=KvXxwC0qbR|8hTR*zy%3gRM|YR4>K6dx=@+y(aEtFL
z+kM@ByYu=-S83l@J>59d`^UnoAOz9(AnmiU&^69qu(^EEeJHZfB+QwiNO;G0Io*9k
z)|zWKQJJ_(^|Nn)#~&+q@Bit`h6C=@J6aNsyXT}Nngfp=gISzZBZ3pyUX_uiM-%l>
zc3@8PH#R!O2j@0_t-dy<s-5t*9~n20ky)8BNI}c&-T3$Z=Ioa_mO*Y%#L+$W0$3hh
zi~mbI_2bJgJm_Cs>COL>AiFC(5c@ySGBfM{kFD(g)wbpMUUw{5u7zU$3<_%ymt{Th
zs9yh_sb>Xkq=MMMR~`#M8w{w?6bD|Xyx=$lkXpOIN;$k%1#y|<A@t4LBL6Vxb_e}K
zQ6nWO=wv7oL%TlOB^Byrpg<Sf;&^DiDd;<)&Mpjaz6iSKle$3G#;y`{7rMZ8mAZe_
z&;r4(z}=9K1Ki|Aj}+)@XrQm557`0voTot7tAaL4u61?zeGEajIm*<Ae)^LDWda1D
z_I@;IFAn|F{=wX6fz!2&^UY1^O}qoLT=8@be{t<0x#x8jo|lgvnIuAOC>7i(XxTpZ
zJArzbVHAbev8yA>)l>5Fxoy~rRa7581YP7}2EdrDGhfxM&qB3EWmQ|1MH@^SiF@dE
z)-}9un(u+_V#d@^B4ZgN3-0u2IU^B@4|z-8ut@3zIYyaM%&puOp}IpeXot}Skx}yj
zM%kDnss`GHkm$96ht=!_SBEpfU$;SGm+{RocD&2c_(P#Mi>h~Pk24Mxjykr{i56!&
z?kjX>I|miKU0gzSMqTA=b_yAG(d45~oYk*20SGt|`4DJ!N&MQI4F$yCkC-|tL8%*P
zcJJM%J)VGT5l=uV@o^orPZj|VS^#9kv~FCzb*o7q(A4O0v&P#BpHK4W5Kp&(HpeuV
zcr`D%HjjBA&(^<)=JL?Djs3k(6j(-L#t(Z&>?_g|-q<U(<5i*8$-oY^IS1hVD6ZEV
zoXw6G4f&F6J7w1l3HbQn9$S=(8VgpNoOs}?*M0Nx3uD=xwdguM*+X5l)LK+I7Mz#4
zV<*kQH{>ge<xr2<4z#abFMJWjePpcQ&1qYL3<IN#v!9+We*~%XQHacuBTxnM`e66*
z4KndB0vL?;I9QasK+jue_5@;ugU)NVzQ&`@pK)i8Q07x}>N>jFnL?f9_SL_q%XI8(
zwp)i~3s(p$Kc_PAyJO(x0EHFhqI8_%3RcF!(rJzxhqV_AeStQ<+JFaL$sgvQ*!`>O
zTi6PL`Wp|H+4|4-N^E|-dnenv`q7{Ay9R}a^z%Q15U=V9_y(GyL~dH|iw{xE_ffu{
z@|z3D<XGd|iE^jtDtVSSgQ*rPbSC_Acq(L_!}W#a13E`GV88#qJg9udhW<04SLZ{!
zVApr0dU|~{(nJnIuj~fPG`uzi)eY#Yp&fO+ni#{-2ClrmykKucM`JJLrk46WZKJf6
zL9I&phF|T)bsI%pg7J4jI-)-Rev0pQm`NP4YZt+G`xJ)-+jqgfzhCg|a-m}f#E~8~
z5eC6Th6z2Q+abSznmcyy%%4x}6qpdaLRM9o>h9c$UY74idx(Eo7?;`uF3$uIRVWDZ
zCn?7@|B2IjvNG`&Q2}@+4kKkH<11B-g3$7i4Biv<TXcR{*B`tkJ1)ABkuLZR_j&`%
zbk~tPC+o$DfPPk<e-XP^QjJqjU%#lj17!v~@?<`Xr!ST(r-9}x!8$wqW>*3WAG8ma
z1V4SvE8?!LU9rz62Z+w5=X11JO?e5skC~~i9?AE70}7XvBacTK+(AU$dEDr8cF=)%
zoW5l?el8bqBxkFF-1C(aWKKVSw5Q~*fD&+nQh!XK4(SB%D=qQv7mWPVP<vkz6bWmM
z3n>}NDX%^}>;&(HnMDf{hmyPgL%T0?&~FjVw`<4CBIbaFKoA5{Eg}@-(0<pj)^c?!
zM(PgXQuBdaquz<<peidMvoE&OYEumh{Ywtxih>o{Hx9(F42ce~f=H3rv()>s5<7Qy
z6f!`}W6!EV!D|Kz=S1|y{_V5+k@DPx*fgCW>}0|Nx|u})cbEuH?#J{}6+OUZ>LM5I
z*2FM>s8|C5&nz|YIJ-WH0qc4e#4FD;yam}ssK%C;!UB63C>pGgU-VQ?OB47(PtO9{
z-yL`(vOvuD^wG_{8zU`G#CY7G9d#!Z4m@E<&A3P56)830P2z1ds^h+BxIC`B4!T#%
zDZ0skaa|e?y84TN6=#ms$eo4#!4ckyJbOIq>naYXSy|P5a%7zt*$HJLhqa~#ul+!c
zZQ*Z+x#CY|&V7tN_l%CY=Y|!k9NI_e14GD~M6*7dt@n_6`I1mgK;5>{RNW8h{dZ3W
z%>{ox+?8(qZc{#%Vdn6NJBX+I2XtNGxSH;VLORXM>StT8k@$>^xF|V%&AN9Jv&A>M
zbCZc*?V^e@pyd@$o`IMP9d7ah+ti`UX~fR&F4NyM0ZAM_$@zC>9Q;M?oO>w|reEWA
zzK8h*z63|{gZ`^y3&cn-U=GLtS?uA|_eoqH^1Y=qJZLd7|IluAkVPKG`~-n+-Bbf^
z27T*KP~9^eXgCe}uAe-Z_;Vn*m7ZN;OQ#sdPD`evbBCv`5zb)*l{!8k!G|=eDYa?b
zs<SKV;IZhqk(j&a=hez7lk+D6ma0<IEuV%=lsPNyC<~kh-F@2|r7|5*Y!2HcYft^k
z3IcI?FU;D%rwH`n5M!PPCKbB)E}{CGn4R-b*^=6JyF5dZBJ4F0HZjThd8tjQ%`~@E
zi)TS~jJ)%|HsBT0Y;Dot3|>wT$d(*y%`!KF5>0EhFO3a&-9X0nQRJl6hSkB0c-?ly
zBh?;X+wTlatgixV^iFaMZSzWXjGR3%X@7o`M&BdtD5eBG_X$^{OH~yG5vjayf8-WD
zD!t4oWSdLyqesj0sl7jiSPp#6zZA=7a`dPY)D7R&=e!wGZ$?QnA7LHVglkzl14nEW
zXAmvmg^z(?l8I0cz7-I2eJkeOo4%3-I)5xLlX%gWERzQo5CF*o=s4|v9Y8bcw5%sT
zj%8v>64rQ$+n%bhYVR$s7y~224|~cr(08}QCf<YIHMbrMls@-<pFJ}ouc7|63G6Bi
z4z75AiQ;vuS=u!xp7jJn5)AS%J}>e34h^4YZ3;(Vi&41$BiTOcbFt$T*gLMuzRu7C
z2y#BFz64~W;C}Y4tfb7<k9GTpytInInMG03vD5F1tY!h-`FIJT^NdaRx~<Ks9K+M!
zp7#v7U<2SaL{CPB4HxZwabhw1V1d88&{3a+KYDK7kdE`^>`&Dfs_7*H&*@*JexRlf
zL){U(6qefNA99shtX2Y!v-_>{!ZdJIrf;7{h2?1-h>wmW`o(?j0^1A|2Cgd48k>vC
z)Uco{9tsTS6mW1&ZUbMt^4`MhHli=A%tV>i7mK4GrCKEDtttn9Nl_EKPv#TdBmcl*
zK~I86V~5_r?&&8g-gK8|&?pCt-j6=*b#9CcFQvplsu?O$v{%B_l&TV>&sT|>gT&Zo
z+3a(`ga0`kph#U~&8ZqKA<W(sqMJg4DMZpQ_k_xkL+mcIl;efs!m;M?DX_~hs@0>E
z@40hJ(PWZ~l5e9+48hoQvOy{Q5Oq#wGiQ-U!E#CLdZhn(*R=OrJ)Jn)8C?;T`<dKN
zX{hDpA6&k2g`2p8g!AJ^#tZUSL~q|T`=$28??X&@4&l+XVh}@zq%$SU9To46p|*mO
zV}AN6nd;zVQ;&*_wPTaogMZ?6u-~gud%43tkA1WRyxzVLHC6}9vun-uH%KjAU^kI!
zwGpqk^3zuh=%d66rKh`1d>g6FaR1DQ!Y_IA)P;)KywR`7>P~015EkRBcH9J&862)V
z*Nr?56lqR>V4hj{Y)X<Oyldct*s`B@t8+%gJMc7m?J1q$4Hyr->8e}x!sZ|m;XT}O
zs`=+>{9>SxmnrK(&t<4AzB&$ex^8(?j__{IWA!oo+Q+jrwu#<~_HXxV>i#?s`hH|<
zU_pm6O?##>AMSA{EPW*_HNTf2ps=u?B{Of=cQo|vu+>of3%3X(jj#*OsYT#^*K;Kw
ztlOwtby$6LrCQ4tEWszkrBhE`yd|8z*z&vNN*%G<h|hmtOY)_@!oxs=W=%mD+Z3dB
zc@R$-Zy>e`abxou3D~If&7uRzgHieUPHA?*HMvraiq?;FdWz5+{oY6fjT@oXPWx-6
z@|~7!rWeCg|4b6jy|8R~f}m{^u$%v3f`83=%Bt~P+;Q|^VZWvL6yZW1J#gn2wI^Xp
z34$Y1VlISZ`xL?_cu>Uk+_$B+GiLHHa)#1r6ESLRM9w@WsH?H>U4|VYCGjL_ivYv3
z&(Yf7T-<=_lLK3|wF?fK@DW(B(cm{^*<>9{ubPsQ)(k>M%}0&DBuP1zwjd%fU`^30
zX-h4WM|JuR_BBllS8wbq#|Ia}wuej%ZRN@==^yfNTRMprR?2pEH9biSn`L`=+Qt)M
z<BWflQ6yoz6?cpeA3CE!pNENYM)Pyy1)l(yi<B(wd%`H6-1BMz73(AQWG}{;W}s@>
zrWI%WwwvcJmT)c4m!ouBd2hAn95&<7@#2-83$f-k0sh9u`aoqS;y7I?{EiOrnBD#d
z^XkzST}-Q_AErXcC0zV@ZK%R5Yv*V0P_0Nhh2)<%DhxcY@$!>*$21qWR9q(FdJ$1f
znD1uZ+35H&?|Age1ZL`SmhYpCY{PS&NAc>$OXP1pk~o-8jCo~l-Y&-hG|0eS`7cE1
zl#%t^5qS0FjAFc+kP4@-3r4cbfRZP*DWQ|zEfNVo!PkY@_Ogau-3cjk54WA6H$JX%
z<Ekp{Xzu3YAXjL14$fa5_kUM?-d1mH?zVgm(>Yt7ZR?t0N7mSRuBU74NNK3Tn8|Y|
zDIPVllCTnl4U3ps^>M_7noKkLf2m@Qrl2=lNV+)8-SA_j><Z#qh(19oRw;hx#Ntnw
zmBj?)-h_*?Y53oq9l46`90Aw3F4SsHBkdz;ON)5YEvqmtF+_Ecz*&#Uk+cqz`hECM
zZKCCC7up75u6+)*+77Q-CejR}B|z>_LvCK6?YMnotxYz_ZIo6%jtb-owK86YY<NCp
zNh04;G}YPzgf(K=Bb<^XkV!8LxJ!whLoWEu=6sI)4sR8mR~ZZERj)Tyhm|K;asJAi
z8H*DcZQCIgKuw#Fsf{{vhar4CQhdr=70ImU1{^*_&)1HY(SM2LD5{=I7QRZKwVC=>
zzgskBT;ZR|;raBkdg-XcX~I%B5hCiAR^j@m#%A*|g^>uZq4fFk)$$(EHNz7lhgS0+
zY*?7ZI52FjAvM`XQGKF<v>7$kAcU>vec1nE?YP|OwRxMfl<JSb(ocM-P+RjnpU-j=
z4w%2&zejb@lp>?FgJnYO>&V3;@q~7?*gQvYgrnYg00nKg!UGZ=IHwHH^FRp)>@B%z
zYxoy<pgzf0n>jsc8Z@qNV+n*BMWV&^raN!DwD@3o;U`c*RMlgqlzOZB&aI52_Ob6h
zyRM5a<z?yYjSJW3Diw8Mo|jooy?T4j*B(ynte*tUr4e666VcLLGUd(+zu;_uhAU4~
z_;*>)jf)Y`Z3-9I?`+W;@X#BR`OBe+Ofl>?4VY38JkOC2NF5bM7Mkc@XJE_)zoVOf
z?kx1IjWeKmo->$?$7>yTzRILS!+(iduc|41k)lBpb?~5&g%nA#im<SHXX)r$WDY@r
zVGSAX%~tS<Zh@KoE2nS7J|O7N%ralXw0Z8D>3iz^wvK{^I*lX-*G=sY<*O5>{z{)O
z<K0#xq(-X?3)6L~&L#h(=VDKFS`t^ePxBgkHTtt&^Uh#D8{w^Yk8VmQT;O>b5w?+e
z6u~%iJg0Z~!NawsSR@<VdvyUuy08<q5S&YD4GXWYyB&!^`B`8gX|a<4K0#)A7B-)R
z4870gfsYJ)P+{*=y&+t>6HctaJ7nUclygV*vm{rzKmF9IotpT}IMp7_`^FP{6Hfsn
z6Bx5r1Vc?vs|D87)c%x%rL;iWptY*(z5}CZ{1nTp^_LU9hdt6yU}SQG`LM}O0#`9z
zYiSX*z{1zk?8{-*mt>jgpe<EOc@Z+Gm8Bg6*l)ZeULjK>x`qWy$b@qI#7AsmS*3zW
zTtQ1h<=3fnO)=N`UmPe5YmHAmndvi{?a5E<JNzTauFo4c%cst*C9xB}&*f{Cc@51K
z<|;#9vYDYYEw7s@(F(m|uT;K*9=?iJx<zBlDSXrRO>B5aptTsuwpqp0q5?T5DsM#n
zioH#!7Ed!){MX6GEn9-ZtF;gB*AY1^^0K8O#17qCit(L-?8b}=eXq>sv@B7K{yT5x
zkWNUHtBxt-M5tu%T+6FQgFkLr!esuX7g6E?H}7pxSv9r5n14=9T;Xg)Tt^y{5Xo`E
zhbVf$o42r|246F$@G>seDyZD>B;?5XM==u`Nw)&%S_o8jO=auG2sv(|F=RP=-hEm8
z5jm}T>jlQg;bE|NoM@0l)>Po^Hunn*!JB5fnDMJ<*hg=4vD?-p!Nt)f#uaMMPHVfn
zf)d8<elwZTxndnhU9q-mJPE@ZNPJd=XzvbYpXHG!S)dyOL-$diS!L~JvWjlbyR&%&
z8S-Sx=q}r};D`)*3h(X7jdhxKGgQQ5Snos!<I8m&6MV#B`CDGW6KsK3bKYqSKXY~f
zXC>Eb!+oP9rWl9b=yI_exyf$M{82wWNOnX~rNXp3w87C%OfxXUKT^lJtlc!Qj#j}i
zdSo(toVw4(@4bU+<4&v;&A>#%X(y?YS9N5oOQzhMq?$b^N_xom!+@uD`q)y0&q0Po
zI)MYbjzkm>isGlvI1!93ioWw>|J|yKnEN(;%RTqD?ryKWV0*oMt)w*gV)4{9?tF3{
zBD>`inEqH$Fdw<Zgw7d}xPNj(hUSts2O-I;X>twC9UokKNa^BuNr6l*c(Dgm7h%v^
z9OOz&>d%QBTy>TGqQomgtqNU<R9Wr$S-**-AV!R9=<LbLFOxaA98j>>(*Z8XX&qe?
zSMtO6qZ6*8raO_=8U#K&QXW5pllJ?<k5d8!c)qMvi^0LGS1j~+P2ZM5B)qe6m)Ox~
zwM}teA5rn4lBh;V9TrwNGJV{pIwvQF3|D1pMnxdp)TtzXzrFTo7;k^Em*h&Avsu1F
z>@!&TO#k&PL9nXeCpM=UZ<2pTS_+gz!1t41N&G{?OXdUrBfjLW)Ld2$r>=st42dbd
z+__ecRPv#hR7!P>vc7At*Ze)PumN!cPIY|ED)e_kkU7j@PJB)LyH98L_D8Q?1HeZA
zfw-|BH|0~Wz36XF;_Ww-PPTaUAeUl2v7UI>P{nWTKy18G%vjqmMPfIVcR@hHj#p>6
zt#D#M!LX|AV5Y2p>H4&ddBXOC=?SIuBrDyx@jXqIxn%0m*5=}1TsKAU;w}1Me)th{
zaW~!S%QP1mL($Ru<K2oZj2l^wk4Pxu+&wWtcotT8eKl;qn*Nw*vu-WNYt`ZD+_i1k
zk7DFaaDzMnHwZNv!|tu6XBx$PzG!hK{$tbK_G<qWe9D_|1GtMY>IVtIXWMG%;?{fZ
z?lV3YDXi{0OLsfMD2RIOYgx^yg*By2J{!cRTF)vlS+lHpO?S;s<WWdOiby;h6)V5`
z)!gasMZO<coQSG^3r0}G!W9JzX|VUMve-=MzyIV#*#rL_eYWTX=M%W-O>jY$$}%pd
zln2}wv1Wdi`nC*h#aUYW<TdyF-aUE7B=s7eOXY&4TB5P<4Gwwk(Pcht`Q@N_U_NVV
zJN;_3hp@SN;RRjkeCmBpTCegP5sqqrLv9*UVKJja^V~MW5PnxT)5?%x-nNq#;Z1wD
zqH#<p-?)3V;0E&sJ6`Rd?OZQdwNDp)yZ=iVbx^<e=;j+nlL^J{>p^halSMfBUF9Qt
zFA>W!aaGkBAGUU}d}?d@3Vh8eQG_~uskM8cepM^y+z*-Sqw0!PI8k~&EsX4^om1ub
zL|!9isS!zdnwZ$_f*pLMf#l*><EgcgGA5yjsQ`(npGRL8XI2M0ExoR7z?rAKcBv>S
z;p6GVaXm-J@pmUA_s6Ge+LuUA;0}KGr;p$Iz*!QaoF<m98LaD(-^Bpm^7rV(<u@l0
zrD>|T;G)~b9@56rT_$pX46!;Y9`G>qjDGvn;p`~x5HB}6X(R^weOz>GJQ{y*pDlej
z!>qJ6U!XeO!?W`I;-ar*in`{0`RJ(>#n|45T{PB{d?IjF?Cj0*!Qg7v;D9{=^4#or
zy0#rV=_A#&D5K9*Jrn%cmS0U3CTe_*gKCbsg>C11iEocUccXxA<9L8vO6j3gY*;;4
zZM=Unxq1Lkv1LGMN9#i0l%>3>0#EcQb@g6Mc4&5O#1|wzCaP@BVuWpAV5#sM8a%s}
zz0Q+wW=LP*O)F$i?OH^4#WSN0naO<kK1V$K6Gb{znAP8!vmrSJ$K#v6_D_w@g`LDE
zI#2A+4p^I8)<2<>D!{7vnz;LlV`FedJ%!Dp=pP*w$cB?W8yo5D`dX28Sf3EHzEQtS
zeX5(U$zu#cnZ)@{2-4j>(jCJpO-Xq|y0YODNoah$6k@ZuQEr2oF0dkrz2+@2SJ@<Y
zzAd*Xx9Rh7a?9i(k$ICbo%sxZ&_Eo_&})jj6N0b|5S-=r>5avoiM)+wbe!Nhd|~H=
zDZYxPvyD}NH6hmBUtP5FW29)?x*)4-epnLe<~sW1=CrCibGBvp_;GOX-ytQ}?lg5x
z25blH2Mgb$-iDE%AD+}J?aJX<-_}#bDkq(i-$tcwzpj{4$^BDPI2Kp5Er>EzV<W@w
zG<{J#DGTm8IcX*66c|!CAKE;bo+HIl%~mS2M<6KYKNP-Sez9IzWFrp?rdBuv)uHL6
z!WEbK`{sq;hL!2i8$s0Q*v~?b>%UH+#RI4tn`yRLT5lW&^Y&nTf8(|cNwm1w7IuE-
z1nUNS7KhiZs}oMMy{w-<u?fFwqgq}&!ar|A!l5R`iE64)I^WU^?fs+u&^U~tap=8c
z-udlL!5XyE#+UW30o+IJnqH0oVVh9!qv*W@vU?3KC?utDM5-_kuG^9|o>kI4_7uli
z9|ch>FoI32>EvL53W*jUURIhGzk?FM?<$gTz&yhXXwV!1)CwdV<oBDMN&fyP0qr&m
z_yGL)w@T0-3I0>~`u`lJ`u}p+JjuDb(BE%VV>_O(mEc6RkxtBicr17e^c9UOfBTXB
z``z4rbyYw!AsWb_LC+In9f7)@0QB%|7=SVU{TZgTj=<p^4~@aNj%V72JpnuAnHT)@
zVJ&8bV_!r$m}$_1jOwrx7U%=;d8*6)F9Iqhn1-zqtnKe7Vd`vQ2ozB$BJw{+!05MO
zG$ow}<v$hx8|}FqpO$#V<h~C<cwBGsjXBA`U;Sd;7aPipTHjJag8tb_Fu5HK+FZ%_
zS~uULs;ju8_k8&`Azw<~Iz%Ic8W~O?j}!+)%gWQe0$%}WQ#shY_hv=yA0BQLR3&&g
zW|iP)seEq|v%2&R*0F;DL8ViF(-#j~7w*7o0jh5VeH?*?oD7>ex6Jbo0L4$o2Snx)
zLDllTzS>w0ivxPWcW4+6ergy3KSFW2I4A(uWNhVndW7~h#vm#@qz)j4qQl?6)-I*E
z{v6M~n7bo%+sb6iH@ITUA(2vAj?xVJa}*&|-J)p5ks9Qk$SCs$L?b@-<G$e6O(^72
z@}g^jr;*;vcb7Xe!vyuGk4MTAfb3m+3bjfo6mI(LerC^GfdQ@$x_x<E80Zvb<+XXI
z3%qhG4*C0FnrtFupDES4tCa}Uq7}WofiYmdG4|Q=XD9deMSD(b`G9FLI`70m{B|^3
zmp73W%6|dfJkRy|Z-^-~LMeHx=N`UOx9%AO0oOS2HW^HV^k4#%UxPwZixdtB#Un6S
zRPW>h7({#{;YH@(WEjLEgp4VW<twOH_YCR-l?n-fI)4$&#<7AF=T*+!afm^AGEEHn
z2KvhoFmgZCGQUvvCzu364*z>I+3A-OpD%ANmYh4gN#f;tndXMhH*=i^^UC*N1OCV?
zE>s&yObw806Eu8qv1n>El*-a`>bPOsgp)!8Lr*CUNhUNt=AS-j2t(<lAo)|S^x0-n
zjuCZV=$%hzpi~nn|Fev8pbj$DECN7jzAI-5G-W0Jub1Thq9wd^DGlnPCAi)leEJ1E
z8p|V_pt)d$f3nL8JROi_%2yxxRcI<kC$LlU!{PAkvBD`dr8dtWhpVEEnt0LXa{6ZE
z%)jqVmeDDqYOfCTpSz&WMI=Bn(O}F$a<d?er0vl>xMD&Dh!(z8sz90nNb9YA{3#b$
zir{~tm;gp-Z~(@XE-q>pcxX$`g>b#DuiUIV=>+XFgApV5sVZn-ssM20dH*F&*6-NW
zFWoEn=FXfl1saapUR29_|K5`NqI1bFW#@C9gC)R874VfgsmVruW<GbE4qP1gk-xz!
z`U>2f0Z9fnU*kKoW%s^Wg)BqXG7mEF_yZ%y2+^A~eCO*_1DqX%-CEggDQ=YlCX_C~
zTGVR6yjT<G*jB57LvAl$HWRi!XgHwp_36&UnuQxc+_k&~(Xq!t1H5YuK+$l(lvUy(
z63F}$@KftN?&|^x85;hgy;*f$)kGr2K@5$r)I>qT?_Qla%~;(UOCxdB+!564oYScy
z6DZNV`niW>Rl{qJPFoYl_Jf$CGwvL+_aFyTr=O!;(*wkR<|9VVa7#CqX0?UpiUc_0
zd;!$+`<$`tL<7~=<VdVkmoIqn2?DR$i@ZcPZ_tD^Y$<p^Ls#_KM*xm3n+;;7{JKco
z@;J9%7_|WWT}bQt2Z{S~pgFkaJ}cMkRu12j9*t)8Ih-xsnD{mhwNnbTM8k+t4)}lH
zlr~dU&`IA(+d1z+?s>Aeoy~M9*M+fc!<2b+U^`C#XduSBjCC&G6uI{S;kFKPX}Z%O
zP&GQXQh1jIyukt=Z^3B6{1y)z3cBByb$LX}2=9rkn9*&{fVaZ;t%W^1zb<#SUSKjP
zfHIX9_`Ojf2V|GQck97VA*$1BmNQ4a?kJw$RWqcrb9%Mud`89|vs7oaiK|VrXWLPu
z=Ao)^wNu<u5j2-r+Y5Z;n``03a6g1%?Y)&LwUE~=S=X}cL1mTo7;snoxV|UUlUzmJ
zBK0%+>QUvB-mAs=C<Ox&12$+>HhwsE!>dtmEMTp5Xd5tc52A_jP~>?N(I(vqP=$*)
zD0LZhloM-9?q_tAy8UF0_}?i+tc&zT99c(@Doa5@d(sU!mWjA`R9?Sr%2NSJv;YT8
zVzmkB(Dlj1^Zi{Xmi<eAQm`!&XAG#GZ+;5q{iV1#o4sV<j=``V<UfE##qgj=%jytI
zv$p?gi{+z<Ak5?O7|+Fiq?+v5%G3o61bUah&8yI&SLe%Q3Raqj;0klezn+g+!eR(H
z9Ur)CchVmB?1PcaxP13o+dtCIrvSik^l1}TKu^XNJ=7mS<idVofoV_krRZFN_i|-C
z%Iy&9AGdgoBAP<Au{`{9(4d08I%x9uKZc8LGD8<@`UAN8+lLB6t<0uMsh<pnmruAE
zFVou(u7lxSIc(N#YwfbW;57tV;ZBdI3Q$V<k+Q&02g~z1d*DGgunq=?<#-3Q^Yn$f
zp)8kA9BmXYRom7a*PN<h^v7yI=r-4tPE>%rjq@!WkY!ythVEgzn5SyanqlwQ|E}CS
z4Ix${z2ehOww23uCtZ3Br!!8yfeaedW+zLS^WqG{=A9QHKnakO$do^&=}C4X=;j?;
z(Av62jP0Rpqz)NbGUcvNrbWLAS6*{q4<H49N!VyHlLr50)((m0^yEe=BkaWz%cTM)
zj?T5od#($iSmyrL)I2R=zYj^aGpPt352SY~e6(&WP$&+f3wqQA3LAoOr3Pju{AIZ&
zM?j#as#c|*F033q_46_r5oYh!TqQHTDUG7t&VVQfFQM#aZSZ+qhK<8s$y>(WU79dz
z4vq;Wb}0L~s`w7pb^LB8Fz;@$x_4YeN0^oIxl<kc7G%3<5;l}>JO*ph%peR6nE}3E
zNKBX^o1OOlCH(iEvH=XJ^PM<qFj?r_r1H-|SJ$`NVB+BUlI%B>LM}64D@AgVdnrMJ
zOXoRx0IB?g1rd(RN{>8;$AA-3zcw7u5q32hiMRPJ6H8rEUh;4+4x@dkiD=OFN8(&@
zft+`@*NdIR=tlaKtVJ7lp>x{jzQ_|Ib2=){Vega_dEvs_3lEj#pI;uZLmAIrtt)@A
z$b(v}1p)Er4&I}PXMxwY{@H=irS-&Bk{ZoYDLg~CyV{Wv`k^RzQPor|v}gtC026Ao
zfLZ{}_kbedhz%?=r*U|xCJP61KGQZv;pTCxmH=U3c8x9Dp@CJp>#9|wuBwV?;}mCZ
ze4ANri@5k88x(P=HQ>-Ou3z)0Zm}yUn|Qrf9QZ-P3H$HHZ>M3y_jiNSd|r9{dELph
z6-+{cwh?Aub2Q@fY#2I0{d_uyi$cE02+WO;n%Q4`V=(Jw0TRM%+v}gCEx5m@m5=D#
zYEy9(aD7sRLEYI!5F#ru)}Fp!J~JHraaT%Gq}e=gCZJhV=!>}ZT`pAWa<at2%9VsZ
zwhucg;tqCOlxv{v^(*A=9{2l}Y1%Tvtj&om0hW>;Kxv-+bXr52d17P~svCBBh17kU
z@|D@RiNj4Bs@ooep+AY?Jg@w>xGBe>KzEaTnu-hUz(gudx0!z`V!x6Pg~SZU-bxi_
zM%gtQk{6bw|M1z@<9;E$a*Cz6J)xXRU7v?tTx`%-KBgfV-s-zPn^zmyM$%yIe#z%}
zFhi%1U*-fkM~Y9F29p|1PEamzEm_AYdak-3)tZ#<^1IS<3UhVC>f;lag#Pma3pF`A
zkvr29JjGaOUpWYLV!w2<z9aN~nZk+&P515}Qj6=6EjhGm0bCb{0*Z(y5F9cUBXe)i
z(wj9NCnn?~n3w((-*3A5_^TLjRbLg#isyCVrw9hmBUOcK6Ca6$Sq1Ak0C-n;n1p&~
zM%|y+)>UD1QI3O*Mfb=X6zyTf0+YX7XHDU=<jt8cmMojC4y3mQYnLeYvM5)!U(xwZ
zQPYuKy`)!-SuS?x6)kAsdLJL0ohGjP;xZkY-!7{Kb}~tUx&9hM5V^{C+PW7_r6E)x
zSX^-kIeK?~w*UOIrD?vS`XkY{o2-{ad#+kW*gED*cc-B2wYg+Wrc!c%tF<paIA^o<
zKHWNyAy=Awwny{TJ;!2@O~Lr+NMgOO1snO^<uc|_dEqHL<~w~fLqKXJv9V;}u*oX|
zHY;@RPD4zmHOvMG%m~xq`|sS7-D7xv#%mCe-1;q28bQ5av-AnB+UD<gZR9NU8wT7}
z=r^NltCwh2rX=(juJsJ5SEl~veio|9s>&Kyp6s$9|7TodK~GH>wY%XyBK1tp7uzqO
zC>lnG{UUI(sWswO(~B!Fi2t$JkDu&8ALJ+Lm>@rr&g9z&?d_L49KRw<?dzQyofN?;
zeDq9JB{Nz{aSAQqN!iPoRhz8X`8w!8kR{tFS#|zNGnxMp|DV5Al4o5K^1!d&=r~Sz
zqb|!Hadtc;vTU>q>|Vmow#6aKR_`9&D{u<aw!f+6YapF{Ps#J<PYsNC0@2zXj&0bj
za0ERUjDs$yK@rpZWh-psU@T5_Ryp6mr2B6XCMJNP(~#9u6ee2iZ#&yF@l`1bS+<G3
z#etH!(k3d)F`s9@S<1HIY1hdl<Z*D-3p&+dEv@#TxVx^U@ZUmChFxF|d5!zOC7K=X
zMf1*GRXVhw{mJNuzt!UGI2g_XzBznE%s9fm;WESD5;Wi;q4$H|l$mm1$4;a3z&MfN
zZ}G_^;G$#VNukobBcWLRp93G8I7bASbD=<43_f9e8?yYj@}(vM|LVVoTm+@|zm@5I
z&i@`3{r~@Al*g=WpnjsxBFUTCJ23-4|645hRZ^1>z6MZ<_@Q!@BsLvNju--QbrF<5
z9ZBFs%m;wO&LU*pp-=+wDE#ywWJfN<Q1kFXzIceL+zGmZxbQPZ1rpO$=rmBm-hnuX
zNoY$}97J_h#OKejw{^~JA6#TwIt8bTjt4qODLj%jO#PN8FnsfdA^<e%EPK-P+<$T7
zgs7>ocO{z+e4Pt8gMX;ESyBhYZZ+@7ot?=o?(%I4rn0#x9*tcbO{Cy`Yc7Ud9g{P1
z(>+?A$w;0?@HnfGm^#q`Z6Y{ic`UBXiVqd|a*NyH6vN#4$aDI+&dk9Gjh;n@LFaqV
zsb@N#t2>W8r=E+!wvL@fwU!@!&aiKKDO92)2^HYk5k2@#^kp87#VtKOqV%M!prT;v
zR^EAI7+;7y9+dw%RKLkRrr+dfG6xt;{9(5;IH)N6=oM?*w~^@UG@EJb-|;kfIpy!~
zc*ub|2i0Vk$BB`y&dlZrt{?BfoG`p!ooDBB=o_DY?&Yp`@(Z(8xF+k`n|=Tj&P43r
z|6KkK7$?~lp?Qmv(9rhH8HZ-2La>b<4Q35y27!B=3>f*?36+n>S!vM3TCa6;c0OP}
z+YhmL<`X-eShxGt>1Z)A3SmgX5#C&J6|Ej%3=D6JR&pnLN6<Df5)3be20y6By?sto
za8v7{&4~{vvWGj}_*RNLnt#I*I?J6QIyDZgmz9Q&Em!GSN(>~AF~Loa15mxiGv;Rg
zU_wA0?g?@?aZoJ9I9t98j80dFRc}CedQiomJ%gsY9w!7qc4gnOUv9uG`M3PvyJvxJ
z<!7niN#7hQYHQ6<zr9^^cE%O9Je)z>yT<C`S-Bn=Y@0E)O68uQy<b`S&j#q?Z`*)P
zBOe%*X^R}Kp4Lv#fBg*yM29*ru)yWdH-KVL-}s<vG#-*$pwS;1%)BV$I^PI$d|{^V
zdO+R5!9$HYVs@`LU^;@{>X<LE@FhZC(KHYY(%4aU?^-%ge+A*?^Xo0h0ut=`mfs)`
z2AMXJs2g_(>dzf{QP|icXv`=sWwa&>GQUA{7MQLrAQh?ztjxd2(-6%9od3xpAb|PG
zBr<q{Iuvw&yRBC1+|sZR)w~Ct*PRet&qYxqu({NdY#PM0`g@uPTUL)r+Q)<9hTr6<
zQ@7Kfsw(fe-iA__VG)#3b=qCS>Ey@Z&s%2)_C^Ep>jZ0DAWTQ%u#Q3A6I5O|-nrUF
zEC?f43R*zMpc3zb#_|5EhyXBB%`An(O~~z82@3p^V9Hm=QrnKesJH7L(XrbES&>*=
z`sDFOg^n-58<nn8<rLeHGIP27tbP?qUnTMZv2U<<0!62~AWvIW&t_xec#<ciYJru$
zII@yHuBI;1p{>+K?aDq>(Jult)sLC4SnX(@6nj*&!$CD3o6+tP6LdNI?$;KZJCLe_
zV>GE+xmIv03(`VM$D?EUUOvtc?n$89JdM}LIm*Q-EZ?~a*;~E|%A35S{VsL|#M#Sk
zM)$d7P1yR+rcN^BEk9_Wh_2(e`vEHk)3}7&Z!s4}iC+1Cx;V5y3#%i<MFAZO2d-6t
zPeR*2e7TxHFwl@ug!#^rndBHVX0|lH)QsoR;6yv=lGt)D+WL#5+aaVCL;)I+$9%N&
zdxIkD=V4gJ0>fLWEYDZ5&=N8i5dNcF#B@4{Y&TH(yipx;kIAZ8;1F8b<0W26VNp|q
zD?GJdoC2Pt*D@??hRAZ;p<Zns-)fZa3~Tn(*WvWIxQBYxu~_FGpjO^<AN}RyTBMo^
ztgCAR^ZBJd%PG4NngebVqv@B-46E-kuNcBxcjcB(Flx-uC=fwV!)^wbZ3|XKPK|Di
z^Wf{s4ZXk6c)E?yC*s{D%MT<^jL5BG^<f3u4uv(Vb?3ALe4F03M;jR%x1(gXUsay4
zN5yNfADL)i=OftkU~6<6xqoXilu=exNwv36SF1ExJ0z({3G6Nc&{l0ENAIYWIz1G<
zV^v%)r%tSst%pVgNF!Dq8j<tNGcTGHybh>dP4AbL$AiCm-)7$5?JkV~U|8Mjfpj-7
zP!jVycnqcnHRtW56-+^_xgsF)j<kHO2g6z&Y_Az@uS?XExjV4%8tvIo61cie0)qd3
zm4-JoD@6+J($)YTzF@~&D2qCP!%B?uTHtbhV{6v>v5N~g$53A}{b)D6d<$@q?2f=p
zQbS@7HbnwAxf1^uN09DW`<{n%CBI_sJp>~24MI!<cW@`M?AAM}Pg@=m!8vyhnWNJO
zm83p(O0Se<Hl+65xt{?P&QdQPGP<|$8eaQ(X$O$=69g_kb7aS3<7bd}5(2*H^f#@Y
zfM>IGq~#?9KCDd8W^xCzfWF|$K4$p3P^$)8ud87g*$_Q6i#ZwG=RpJaI{q;7kYFVt
zWEnJGb>tkHNV5D`s}2CEHiLR^>;p32FAtVN^*1SopsAK82#o#umUOHRax_w<B!7T=
zOKNAsy{8!~<|v9)ZLdN>sYzR-R6RoPKZ@+%u^-tVbvdfT=<2bP2Q|wwd%>+GUmv9R
zabw4i3*F4Ux4Q{cn(sCg0T0|r)x);Eq4E@f$Fi`1JiS2K>gg*RYfjc=%j|Thh>}*V
zDZ>YnrN+%AizEI0&x=`(%89$&&s^N~wr|bWjDI@UK3i*0>7vtE#D5;#yv2vb3q6%d
zLD<h1%$!2C<82RUD!}a&jK4puW1l7k?$kFkq<b@FdzN(*p;GI=%uXdab65e4>BEjz
z+bs+Ql$Vyu0oCW?uW*Zf=O-maNp_GM=?;ioj-IJE)9nxJrWU@q5;@z8lCYke-dut}
zbl90zWApDK<70!(AQ%Ido1&2i9zMFzu!}BWhL<i;g#c2WM9g-YzGxq-R%fzhu&F_1
zgiCMpD_-V#F7s<(uc)kH`2N9Q##ssFdnF<od9T|Q4KCnHdIuqNIJO&smZmW!o`c{?
zAV22ID|xo(b_&$`9LIM&)JFw?o~Nz(&){2J4dgY9xFtYP1Zy^^#-6>`t;tLS&gpvv
z5b$67rY&7;2x-GxjW@>{hyZ5O9_i2Fghv;NS+>HfzIA~R<8$C5Ep(c6Y#Z*7XQp=I
zkz~L$Z}&L(6u-a4##nAy8ch6Jjh5$nY~=oka%T&HfW|VdTcj)RuS)vM6LC4tGW@Se
z2phe?)RXV<`J}|!b&JyOiA>+T3^Jh6l|B`X-2?MedOh6zY*a>3iPPNn{g#>T*rCyM
zazA~xV?S?52eQp5jdm0?T{E0jpf{MP2(+kpknaoHTKGRIKZ4bhjTEtVPn-Bh_@9x`
z<GUM`^~S(^UfN|Ux$cYFgxvmmw<~9zp|pGUwYutjpWvqPFIUj8UIRAQw)zd&>3m_)
zVvKv7X#R;x&+-D_xp8ejY><u?EsRPb2Lg#C2aD2YVM_dmoGWgxK23GMC~LiDJG9-t
ziIIeRnFSOJOPgPjw?7qqLfr(mIzAM{#c0M^)8>ss`xD4vp2Aarcs%IX+QNep*o`n6
zD9xL?I&ZCMe8T@Kect^hYdz_rsSnxG^?5d{%PQeBS<Vxi&4>0h%-bxDJF#F<x9GHe
z`Fn1A{U1z_fd`aOPQQ<kMHnV;vc2WdC-=ZN{&@blAaq%vY;2BcE}6G9)XFdL_wY>~
zdCMY>@A~acz*Q^~Cr2X(R=;7Q1Y#=J>M2#PD!l4pF$AA-&75tx;WpHf_k=E(^r1oW
z1+}rxucut0nwNT&=FGan+HZWf6K6_cNyosBcy1kyX(Z`!=VWwr=V?wqT1ipjSDg?r
zvLRvZSDsM!UKm!J3PHGDii#HdlXy^R1|*w}z~dLx$?K&VRo1+gw9BTvL8N5fjj!PE
zOqrvSf;CgOUse{sWJhxsI7uf4|49Ea^GAjLUeBpAqMtt}$@W#|vTH(AUH>}Bq~yD^
z458gFsm{F0KLO}FF&De_@#Hg&n+Cau6o)538#QZ|ZQ=Qf$S?SU*(`I~{d%r7SU2CG
zVtZaWi&E(fsy{(jZThlvvKYbmik%~5nu|o*#+IJMUCwmu^6PzJI{7B|*oiwQ)_hmm
zRJB#Vjq#l1DSik^AL7H?daw8YVD7!6;oARy{}f4x2;xLZbV8yA(Faiy5`;9O2N8tP
z%Zz%WcgaEYQIe2E3&ALZV3;6`-bNQC7>v#+_r2wO@A}+(*ShPs*6;Vb>;896CS=dv
z@BMzi%Jcbnog3eLI4#8=;A)7B;%`=qiQ_Yb6F`m<iutY$Gl~-U*2GO)BKw-347C~3
ze*o><YP9n##hA5wLSb<W2{ti%0VySlxO&=X^lnO?3T2PT^aG$l)U|k>G#_&PI^D|k
zEE1^eS6vJ`Yd*<*)__h3lOrH&IGKv@Xscd4coRYu&}kl&S8u>`AMdBqBgIaAy7V=s
zse}FwA8qz0?X$2j?59<lz-n~pSIDC5-YNZAZi^vfM{RTMQXVGwYuL?u(Yz@})nP@~
zb}sBVd~4OI<zuy=<5R-fkv;76j2NOA1=i*xtm9gs;Z`;3!^4#^U5YBN;*-DIo3nO2
zHKUG{qAz=s>X)1$2pYxg&Agc=Or`>!toynt1)vc1w6iLv@NG*XE@>dcM(T5Q-)Pjc
zPNfY4G;T9;xsu|w{F%bY#pcBkEu-OnOuHYgT1hRR_CYa=j{ADQn639_(#qN!fBWua
z=kH0DlLr6^`gLow%l@B;xr4)ND5b4<F!*~&HCFMVhs<PMp8no%#|dTw7?}nUf_3F=
zwc3AmYAB$sAv4_CGofl~D4n*>6J<&*YdSvb?uqWKu&mFIb-AdKhs=b|kv#%;jxcw%
zrCW92tn^FLKzHd5nsX__+S3Gqjm|}h7UQMrPS*_;&KVfUstP?%b_@3o7}{Az3#M-w
zd19RLFb2FUC0?gF9T8IB2Djs)J8K@89>9#Dl1Bj=)eU+))`kJEHo-Y%zz&Axw!Bd<
zmyEeJa!DE?ZZRSl8}^u-M0YOklykSr-IQ~onX_<+<Gc9l*9eF`YA*J-s@bPJXV#%4
z{iMzonYv-b%1L|8e6c=pOV_mSA`S5y0OSpn>+=1+mpUOX=0~(M%Pam$VUWlF)^SZ-
z_8BFe?bfye9#g=_?ruj^m4Ol@)v3y0E17}poN#NlA0GZ>E5AaJ1Lfte#xHxKKbaWK
zGS-|)^l(zxcQ+AMG3HmvzIxgh48%!*a<t8J8n@}#MqLSwiNhL7+}IRAf(qc^wL@7C
zXem;-O0Z*QlPxhvkU(`8gq*sLaX7x3%C^SSqbAs>7a-O{qwRc%AC5%ON~^sqn<M15
zR)tAe7=rQ2>}zH7S8AVYo2UhQf0+5cu^~oS*LkC?4+fa8y=Wht_OUEElQE~7!aNT6
zhzs1MvKKa-r_sgwS4PEIio#8^UYRERmY#`LW2N-t5VmjRegeCd8$f!R6mll}%_UoD
z{I~JgB<tq)%dp?{vlN(HNWu)!Gkd97KIW{|YHH-1Y|tG4^ds0+HzWh%G=^^cgmvcD
zD#~aS!+c&HL09ac>DeTq3_5Z`vD2|iwIMGkwQuUGvWnODVE!f+aI`8XtzBmj2>MI;
z9v%Je6&i}fX{1DNpBit-Am>k`JI~OBd9^PsZA5LFF^eq(jma%qPWqm;C7kT`qRP^`
zOe(XM5T>7zo{yCwtUSKxUegCli!Q3tF36^)xZ9%Bzi5%IZ;F-;cVfsDU%M{v30M*N
z4Dap)h^N9v^?)31;9Z!$2(b)w>z~rv`)B(=544)_y{+%3uvuKO)N_nQVXU8xbLP=1
zH??*b6Cy#u*d~C<Q{-3UE=Bo0LM26+D>6;kW&;i{)Jeve*p1n#eD{o0&u$uhz4bj)
z@h-MaU8u$<ES_&In7I0d6wxLsh0HJ84zhm~KDX&r^kw?>*u(@O)FO?nHJ|Z;`SAwr
z^fg+Ew+lt`e$Cbgx^;Fd_xo|-4@j-TT8%;VkxR-dQ*I!qV?94@6KOm4-0~mNP}2Nz
ziddmHuBZ!xZ|1eVqKVj%Wf8g<-Q~yBtiMZF(=2{4>KHQb?s<0zE)mO|8fBGnv_F?8
zonoVBJ)O57b*?bFcw9!{LOb*4Wgzc>DX}C@v7D9Uz7u79b@f7r8tVLwG?h!DXXp7)
z;U*}&)Zg1ZM2j`f=O5b}Pn|A*J*xN{bih!TPD>PVIP!`WMPa02eP^8lh4E3Z4J@ye
zjtV^Sn+luP=P)&A_~P0`D?bpoz{l?{7EZ57l(e-V-?bC9k~B|xJ{-g&YGEkbr<(rZ
z2AqL`_RW@aVTF=A)7IoOv?`T&T!P&7qV+4V)CC8X62$KiAG!gxti|_nm1i_JhQ>v=
zUF$)nb=sdDv2aE7iT8;FgTLoR_f)(;l_{jndA;&J6*PxJ51q<s;;=(|6e`YK3~?=s
zz7uv1KTQ5ebTqXlen~g7S2)SFHC*N~iXq<iUrC{3AaA4(x<YazG85sjZ%e>?)VP1O
zs>*+wcy)8rz42@7m~<le<|BZtG{qtur2HdLOwA1LsXi<yLy({@FH1V`s2vZcYD%X&
z7#ACLM%Sp6rsg)K6M3R-yW>M|wO?+n|F%?gIAR+9+jFV<%!f1i8?4r5g5GwnRsl_4
z<O62uc1x+c0~;2`mpQ#O22}<R-?p--Bzdr)r&!?6=d32qY+SSoUI;n48X~oF0%1tF
zP0%Tdou<h=Ye18|c6Mua<{EDe*L#J(Z6+?)ntyk&>eSsrGQauM^&Ea8v345qoPgvz
z!}PtfeVS`j@7(Vju_tR2tlpHrV$zcx_I__9U+U&7%{4TxtuOoQh7zF{CpPM;E)aLi
z$e6SD{ei@p)e~i&&va;_fX+O~q^^*@e7H<OsMSy=+HOD^e(h}OfxdEsrOMpG9L9DT
zcK!W&neTw%&LfE#7vWNWhL!d(c)&PuXVBFVuLqmv&<P)(KH3_WykJ@2TPSF^A}<$X
z;8?fV_K*n-dX)`uV!u!RtT@gWdieK>x97escd5VD)kLyo!pgyPsC=Q0=C_$~?f2x$
z=wjh^2PD~Qg`Lp-wS5OG+n_2kVhfnMg>~xet6c!UNS9+0MKi4jdrz};ws`XF=;YZS
z#eY?O<T`k#P8b8?Y`lgQGyDVx9O&LWjXbC36Wwp!O?BEMhsvYnvo>N`eveEp^eE)i
z9aX-(McOxJdH6F@_^;Wal<`E}?5&Fg@O{973H-Zu8kNg9;Ykx>nckzr4@;L)q;hnX
z{&Bh!HbmdS1@jjZiLwJfo_!J#9G*fI{2+)_-+LW}jL{xB=!^^(Lw^nKSiO(SxtRK7
ziUqs+Y(9e{1UL9VsM=1VOyL)+O46DdOiA<B3H)<^Z{`fG=qXt#I01y>pfJs+1m0pd
zP7A&b)3Imvj|?H9R+K1IXq~9nUa{ye4o{)u<8L^DMJ>7p!mvQ&HCf3I3by|5@uWkf
zU6^#m>e-N+m0W4-wG*HMt^~wiomrdm+;TxTl)Yyx>qLQ+EJ^PF$0GQA=iVqZo^u%t
z7N+BqAN2!8YV(DEFOgnC-`M%iF>ndk`ZyY1%m4uPbx4>1mAxcpgXtP%xc+<_0L6Yf
z1C&bwz~28H5;&<Q<%3!%&vC0S@Kv4Ce|?dx(>fDCRiJNCe8UQmvj=tS7PoPrj{TV9
z&z*EzdS-ur2pnJRkRon63@WPwaam639iWSA*x{Z+GNalsxgLD_1g%0h+<mO8{e7c9
zx4pRrp+^^o2VU8S9N$?qUxzfFh6za79O5sxh8=F*(?+xT^w-95k2_Qew|~xR=RsLM
z9)`LK=00K$K<rPeu%YYwaqzgr1!ak>K=CyLAPo5gGLE(tjM94wDQ{fBcT>u9<Kw?_
zgswY)ACG8c2U-y8rHm&w(;jU5HNa@w3VHo6f|0w{369bxaG#%IxOl)%ukgSYyM1Z_
zyXP(tXNdsFc19~0`)hm%L<WhFUADZj*TSz9lmK0QPZT%~Lnctc+%%W_{`yO@<={@J
zFq=hbl15;ekP@U7NGBFb1}!2Jp!$s!fZ+;4s*6`ww0ESk`a4)|n6S@*63ctcl9i|+
zq?Q7tDa{Z}R%802x4)56QNPT=N@L&0jDDxmX~baoASB6ofEhjDK|0|K&t|Ve5+%r$
z9J=MrL<AayXMjkPF>HEJYB(CIiH5tf7h^!1y7!<KK*1)hK$<K7Q1Il=;sOz#88PGG
zxPUI~IzTh&Lw46=i#Q-SRrrD8%D71f2-gp@;zsvRK!Umf&qvZ^aK|4rK70P!{7g+1
zsFr!nfI7wEN`wNA@pv6jUwJVCQp81Qlr6HJ+EmvAkeR)NihX7h^xuKSSr>8D^igm%
zpiwW#d;8V_5zm(sPj>uLx_5v=&}3{QM>+veLCPE?l5H=Lp?2s(-@6<P$jHi5#(>&N
zKHl^QL8JMjbfx3>>}UhW2TOm3po5Znq_EB|RNTA{V#S3wh;lQCfNLVGD$Rit%yp%D
zLN+9Ov^jw?*Z>iJ1QoFbnxW~vdw5m*xbmagT!k`qO`O@Le#qN3dy8#oj3&fwd?k-Q
zD(N3CFiHgFz4Ali*vU69<+pND9llsCiZGX=o9@&doVB_|!t_4U1W5H6!7|uFmJi3_
zYUpCNwn8;u!R$SbmJLgHtGyYgq8|(~A8H{^bx(Pv9#m2@2`hz-NEW|&;y<8X?V(+R
z0#K4Y1Mf0;H%Euj=y1x7JTKhca;d*x<xbiRR5?N_)0?$7Lt_L<BiK<$p;yvyUho3)
zHe8yjGX3zMM9haYfKz|^;@F@M#sgjllqk1k0Bz61>CRLIXkCG(rCsLfNCzd?Jn8tk
zO-T1gb7R5^!aCR~n(cKbwt@uaX%9~?GJnP$ORxTfv<9XcSLKKSl}oSgrSb9A5b54&
zDmSDZj7C)L1tc4T3fH?p-6+euSMBbxzg)dyia#ex1S5I7N}xK197z2bg6a-HhwabE
zYP6kT*4Y&#-uo3x{30YOx~!-DMpNfd7#{G%8v;wI5}hHxk%t!;il9%Jgg_&X-UPyz
zd3`3N@3Lz~-X_=BuOp!P$(#LL0E+UCy<W%2wBI115H0xqN*-Pc-k<)eHcP<H5e0QH
zN8wmGHfSQZ@2TU_(U9_SQ8FE<2JI@3TQG~+ti#GiWwqTaN(`W?un)vdnPbHH6mgJf
zxl3?KwkiQHUHmb$;slK~XyTX=aX_zRMHJi{gb4gxuk-x#OIigaeaVOeVPhe~5%Xj)
z5&YG77voR1lLKS!2&h<0FwuzHFph)9HEf;n?JINEuixzLU_8sn^OFQ&(zT7)Hrk+>
z0b8kqKRgyb=k`mv>)$bQ9|H8EHY)zKn{M#xKkry1RJQr_9oq~kECoJQkr63Wp?oTo
zXHUiKxJIUGkUK*ap9CAXHJULnWs%;YF&_`BjR{mXV?-U_g~u~sd_g`+hJlys6*p#Y
zAun$`@x;T6+1(^Q9rg)mI5h@{e}v5Tr<n4rU4TZnz#?eiO%hjY0>19I0wD9Wfe{I_
zaxET<YPX=q$P8HcR+|)UanE?u`oZQaw_9~nqW#w6UC_`mJ~B$aMcD%(B}_j7phex-
zls@0o+T|j|9Uwu;4l@bJGg*a%tM*^(&z1={y=C}e;hp-4|146*{Ze-1>QmEzn51hC
z3CV-QKqK@D8P)<R)j&sXSF?aYj06eKOlfBbfESh*N?<!h6#Q!rng?F6NDK6|$8D+z
z``N+%<{+q~qj%;IOgk|m^loxqFv%-+>u3(bev|r50BuInmG1iZ$GgG06&(wts)3rq
zt;O21swFUupbQ^r&Lo9>U$9aLCL36VUwijSNyZoJ-pynp?P);yiZ~AIXJpwUfGNwH
z3q;TK54mjV?{z2%Z^~Uw?0#3(7Ut=4-d$ScDZHv&(9v26_&uJH9E&XUOJ2_|P#wJ5
zamadME|d1>$g}_(XL3p>)zmm3oivVR^Z)b!HBo^}h<QX{(h4R;MSg^Dg%NdvS20Z!
zOMf+p?`=JUAcs{+>`exj+fkR)v2xC~Q@q!ynC^ZBS5s6p6^yKS=oKXMb{y8#%_UBy
zVv*okoS<EceKTm75$+dvGj<!0^GthYXr4Ok(mmy~U>^cx#R_&p19zA>ZOx69BEQYN
z;SNlswr-jKWv%?SMW(u)JDVS*L5FZSMn464B04vP-WU|0rw1KShoh2{cXh=h*{SK=
z<3;H|k9G6!hd&9(?(6fUrsJcrrC$_=Dc3T})h_-lY}YujR?&(=lt<6KGvKV`>6ScI
zu~PhU!J@v`BEBrNDPe0sX@siFqurQ)>{t0sCwrqOH>sI??<*uJ78HzJdp_JU`j;=k
z*^jy%W^<h)*msH{<5T39!b(falZ;nKH<Sk^p_Yh={mvp<>d}w0k&rfl)KtNM7w7c}
zzj&&W{5|ayX{)kbJBf}8yN>riT$lU&n$r(VM=%r|62|yqR4tsfExNCDunCU!7I*VM
zfV}<!76_SeMcnncX{0WjP6eGnkbts)F9WaG`<RJ`2^?G(=0t$}S~5O6A^Pi-EZ?I(
zoK(O+`l*Qtr03AM-vV~0?D$SSMKo!q?ft!t`x$fTOsUVGvY%=nXKbD-Dv^)XPqfB_
z@75y9W$5%Ky{Om*q0&TU%=8&gA;-@!q&NWy+ix9YywRG?_KSx#m>+GVZ(9VqJDbKT
zYRO@)RxSnc^$j0`SED{Y-b!XB<ttGZJujrvu^b3eD6OV2dCb&Mm25|hIZyHd7?9pG
zx<+AQ>{Bc+ld?q0OT|b&fOtehoU`qhYH$2)sG%c64@9OJ!A`fnWp-^RoqIScIiKQ1
zA5ct{ddLI2SJyYNx!xrNVoJJZCQxe(l;Px~vP7w8zekEuQKCsTCto9+E|3PXt_;|~
z2y65=Lh;(QAXc6i-=$6^+_nX>TYK8)^a5M-^4*@1X$Yx@mwu=nLKKe01)s6ID5E$y
z+PHxgyL<m}qaY4YUk<tO60v0mF^L!PAJ#sfbCfR5cPMAr!sfKpAW^BA-7&2nebz4T
zanE|o;J=k<j29e%V4>QpHr#gLHi4nd)OLVfuID0V>8C1+k)4<#NfJ7MxQP%arwmq5
z*kC_$_>v-5yv19Fq_v-?UWdQX3U-H3w(90MQLV;P(Ilz!==qe2m!7nHIU<FYk%3P=
zh`gAlvf{5KOS>H3O%7jE4(IKUi^%@%GVS#j)gj&T5NBY!7O`Ag-eCdUz6a5ij$@BJ
zO~bZ=;1?YBK8~l!^D+Rv&%d#v7VCA&uBk0?$D3XJvq{p4bB|7j)1y<7x~T}pR?b4C
z3~9+jP{)=5A2Tz#3~p#CmBJ)tgDApVf%R^wTx~dfs$}%#N`!SlXl7YPV#54WYfzbj
zwT`-c7NpHeGPihlefFA9SET8aesILg?OrmPdKE6lMXho@$Y|9xB|csAmz{pYQ_L-r
zmfDTeCzoWH-touhR)-Jk*DO4J`a)8tXb;J$<J+suDs^%h27g}N!PBqD90h9<ZtFp6
z$(qR1dcNvmT+gp%zZIWr+51$MQ2f(#1{B@Kd_Dg%mS`}&PU9?V$Z;hcwDCGGIM){@
z=(9vn@x2_Cv~etVDu^FfR1(^B(*G{Gl3G=9XhO=bvpLNdb*jM`JIPSGS3aHa)_S-x
zkMX*e{HxSL#5>b<P~DzmJQ`Uuhkal?r{%Y0RQfE5{;1G-{3=`Ot^YbEZ=jSp5!y`7
zT579PUN#-6!U*rb9&iO;xK$hqBI5k>ZHy6J-QQw!<C8$;C1lEm>YYda1~ya9%x{e^
zhpqAIJc9ixP7|++gq5%}Zj4r9k?Tuq;*qg%$Et~7UPQIysn;v8K#Hj8pqmKJ6-y5C
zpsxMRE{#>XIZOoXL&sBxQ=e*NU++BP!^!I>QR~>+sXFqLI5{5LJ#4sYn{kde!Qf!H
zQD?2}@2_>iou!L>9+fjU^3`gn5I9OD&fYYBwb!`X8dKdWirlJCf+U<J(MvwjGa-x^
z0;h|@<ZXE;7MH>^bUCiXfV+xL4Kp8T{u;)8{or|3PTh^VDu%7SKBpX!F&&V>7e+VH
zO_TZ#3nR<ja_dihOuiFNWHep^rl5RxNAs(NiHgG(sD#Ald??>#P)1q<l8LoWPin;?
z)xQRoSsnK@2Ah8s|GQX?NEZSklAK~1!iDv|I%Ivg6Mv5u<2I#1wjG+2!J$`(gAUBW
z8*FphjKb+#=+yFc?GU05khqT@xJRA+YfeyjN9bU(^`(4qyWqSJl4{+C_(EyLd%XdT
z4!^C~kq~N`o;zyD5+5s_@O-T@p<WS5I)Rq5Ck_c4p&!6gUDF5FdRvB(qi%htM?E+S
zMc`)JS0sOlMwgqNb0CWAf%ZXx&W}d38P*nK@2LSvwiZLgJqhk%lA5DLq|D~yIlhu=
zzKq$;ccx0U<*8#D83uNivlqjqt4de)B08BPa|S=4|9sKT)1V@Pb~UWO{Hf0DWVP)s
z>`@k(yf<D=Fq)~xZN#qls~>r5QZsz{=1Opmp4{4ApU;ptPK|mJa!5LGRUekpK*5d3
zu#w6GA`cs(+u`?Sn>a9|&do&{^sYXm{z%GvUwxLH+kTR(_@3RL=sr`TxeRg9{Lv-S
zzUVrcp^yuMd)~SE2*+3OUs*NKkkR9dZ@~Zj{V>GEtkd*87GajfH_ZS0A5D(WyRtvp
zSbmwIkk8cc>qCt@E_2xTCfKVRdNn}`esniu8qo9rl*c<6d0I7Tgju&{Ox)5GltgHu
zk_ecT)EP~eL-8L)hGV_o|D__o|J}8%roCCb3h5e2KY$7h((WNaa?uJ<YL6afQl+mO
zzprNpX;GF->_<}0Eqr<YfnC?ZvP5&rOGxV2;M!jC7hYUxGKisdpmwXw1q|;?47@1!
ziE$Z{<^(TrC`sO4GJ{GgKm~yug7g)y&#^y)diREjhsAiwRxf}^j)(fRR*MDbLU{gT
zt%#DULI^Zz0Ek|#z;|S{zQ6%we;~Ol6BlWc(3uCG3KP?{e-h(?PAdnqU9%z#iM|XV
zAT@}gm3T8R7n*5gkZrfoG@hpG@q2dTPPpwJIF!ZB0K%pah|zUGel_;E?J!5W<}+r|
zWJO>AMCLsdN14d&wvi6DLgRzT-WWSF*$Nuybe!}tn)#4Qfc!fF!s(WiKoLZyz64O3
z-p>HvoRHa015-$=H$Z$PXiE)!NCOrP_ai{tdRz|58j#`c328)Brw%iUVI8m`8!Emx
z2|Ru(_f8gwi8QSP*@%JSR&gv%xCE*jUAtNE$6g0Skai~_R<8@BMFY69ya<&!X#7*3
z>S*`&_7=gh2>0oy)0}L`M2{|1uNs~E2Dm2cpUQoz-2Yhy^SAWwtV2bySgy5%ZSQfi
z>E-yCgnl@BRBw^ncRaK$6*ug>Fbe12bjpV}e(Y0tAiIBUtlFwcg<ALtG$5-=+h7K=
z%2n<I^-S7x10p19dCCapnYn;|La(?fpv$>Ce*@;XUMZKfW7VkY%tL7cVy?_v&@qpC
z(jmh|v$1%q?7=668B2e3;jsiZ?jL7iloBair?<iwrwGOo&H(+vgUg>O2#~8~4CK)9
zkYf-76hNYof$Fl~v9uy$FaHt+i}?=7M~csjPE`Y8PXl!#q~^=<g>vzSEex8J>_8Z_
ziD2*b*yoNEderkfg!`8A9^^1QRso7(u{HbQ%fK~Rxa=8oGX&&2)j3TP0Bfwyv<0H5
z+J5Eq8L0c#=LK|R7t0PQmH}njcnsJI^;QbkmO;$a=Pe|>Ij!CA(7JKBU00V6K%<bG
zuIM*iaXVQPTRS7{_!-WduBSJZa2sxQEeHrPJ3s|k>2Swu9}-fC1ODYe^{KTZ;4#vF
zXYA4<4uk;-P<kY3tjdBI1%i<A{*v`(=&`#4QDD0sVHYUR3dwFIz%*J65x>jQ^+A;q
zsvg8c`D&Rkm1E>FHEh`of~N!B?WL1!&<oSjP{D`?h$n-~iE4knR)3<b3`(BA0X}{I
z<A~3C`%`23z8J3MFogu(3!n*#7kEOo44im1C{HN4UZ?yCjl$c+08S+aGjT@rlh}6v
z|1r3n)(uMOz-1b|3c=n~*;44>GuNb=<mz^sL`4Mizp4y@1=b2`WY4a4Dsq#6ozw2v
zvgGKup1ROA^Oi53svkILOJ{391ML{Dm$Unx<KR<-mZ{t34wMzw)NIjrc0ZD~HF!ez
z5gLS7<D@3>+6Sl#dtE#UddLq2`9Wna=niv^Uhfc=m!D3V>_J+JJg1r6r;aFVYFbnH
zOq^&|4XEuVLNdhJ=_@j3UJWWxSxz8H_+{8{NWNxj#Kj3ck;P&Pj2DCh6SxX5eK=Yq
zOl3Vcw30#u7DZL|HAO~Pp~>-Hr_o96OVh=S1}eju_L+oz$YFZthBEH4)*Jf9hO?9=
z77^<DRoLr}H*{{1f2b=y0)@=fh57`k;(-n_SSw~Vl?@b>&*n&Ot#t`S-un;aiBEzv
zEvt&_J9RO2ShlR)1^v3Jh{vbmc8(wy!s2`9vXfl05Fg%0*{4c(jjuddlMWkHQWCk%
zXEAh~QGcF&o$!tt=ek&bKl@R|4=}LMFqUX`d$T0k)97m6x}E8IO3Co6ygsi>3&Uhc
zT2%>Ta(OtT+=-`*jXz;FHeFl_Y<I}E>eO%gJ3#f%47_;0uS|qYDLtCKwdstO2609J
zq1!e<)c&5A=W#55ZCwO1+65|oMXNc5mF?t^t@0CHe#bAi?|hI4x~7OHMeho@OV1&X
zM=PXf7=K=WS}Tp-9U28db9FV70O^eueq(^T_R9`~_x=#xuiVQ-0qY4c_x<Z(?X><o
zjZbarKpiUY)Y~5nftt)~Ku!E!5*w<xk!8(_ze>1_U*I<59ztS@*g1cR&0Mx-|Ktst
z@kT(qNBhK?GW*ewkj`@rhA>w<C+<g-ft@sBEv-)AH64IK#zB2$s9XLuheu1*5c2RT
z=qP}_v@Yvr%%SA`yQlUfz)sZ<uIfCW(;ob?wB)`S`O;jJ=QGNXr>P#DSz*?Zf4{TV
zhph+54=4`2=7faPb_PK$rBW{Jq!q8~Lkdar4$UH8At|Ad(!i4K#j{rDq1%&02bNFY
zebDx%CU9N~hn?Z7qEsaZxl&%`v<UwOO=~>=8+65PhkwfecIc<ZnsL{8o;L}vvwB*-
zNkz;S_sIwdXF$dG^R6@Q`%oFb384>@?^lfg*Y10HlW2Z-3#b}vBedM<7QP!T?s!yZ
z20x*zQ4yzX>Yr#8e3|4l!v$pSzlWDE`w9DeNPSmgr$3eKM2xh?$uC2*-Dt`>a|-e;
zqGs-VU4xPHV-7ZqD5>zyMlS{nO|anv(V|*}ULqeVxW7Vii!{qHR!GHgMI4-_1Lv-k
z8F^RevV^^aYuv78f4`Y^e7c33j{sayJJ!|v`PXWEfgeg2^&*(>SJ?BZWX|c`YhJ<P
z`%BhJdA2gKw|N(akuh8&l(*QQa~A;yms>!}_5PbMe1lZgl)(_CycJzro_W*k`AE)F
z!BKP{FyjpmD9zF55vfhL7=kb*6pjECrKu}-*GD2tM}{GKDi;|NoTt&Y3;VY4;c(%4
zNQEdW;b}$W8(Ma&ES}c;8*6I}?X3cQY1T%cf;Vcu8Scect6VoFMd-&Lk0elkZs*Vb
z;&)4K6Ab;8#un5(A@^tb$7z_Z?PR7FY#g=^+bUEbfMB2vjA1n)N6WZ-vMK^Bet+rR
zMb+cCfuRw^MiksdMV*ngC!Sm<38*jzU7`D)K^~d=5{eEt1bIiFe(hSq=Aa&nn1~8}
z8gBiDFG?#rVQ{7$Zhkd}X>N&ladevM#mq_wytK4w`O?pr?VFjQw5gnW{6|qiZ~kH!
zo%)-~q9NU8DFl9d6ru+R3^tL=VIP~~?7TH26HQW86ewTeHVJx0rC*uM=?e1M_3U@$
zW2KGvf`qt)9vBRF%&{@zi9W?M!(_0ZxJMKC>}TGm$G=8+z=Pa0^nJr$q1>_wp6(0`
zBDd7F$Us%uve$o6Wo*+3@C5Aa%Qy55tP-jU2#0?T_6VTFRmNu5u>lna$)$YzBzaM}
zgwH3~H&c*PT@J6v-n*3(HW`C1O2s=_m89qPt9q!&?$OLM&&fC6;Sx%fuE>yHiu0$&
z&Cd;Vuc5Cta|+vT^<4d&&&XEp-uA`5RmK>g2lkic%Z!j|J)(ic(KQ*IA9D2vq!d@B
z)vNNpoli|;TXfa!Vv#Ggt1SUK1?l)CL7qZT8W!8S!8%-wP1;cS>xU*1^2Ea#<Qz-)
z@5=efEcMv<at(hBCodm%b+f_8oouz|)0V{;9``Tr7HC)DXa`USYo#kO=$p>cw+C@h
zJE-x+{<V3dTH>JJpWQF+ilL=(#9X-YYon;bn-PfXxqikHliXv%L>eyBcARGV*C^<%
zfHA>}uo8b&cvmZF9n4b;vsVR|Y{XI>?+}+@D^LdzvJ5dmiOBJP{l7zM9RCWg3lMpO
zL3z&_{K}ieo9~bcpfV-q0vZ5`;E+5<ZuHR2ZXc#8v4Siq>Ma0gd`bh3ai~uUT}8vK
zzme(R+0^{+!a{G}-T$%g3u05q?R1mo`}$8=sj$N+kd{T!;pP1QwT?GJz9Nt(LJZ7f
zUlw3%14-qJ|MFqkwsE6!IT?&P!WBee+y7EGq~HJy_w9Bq=%+p_F!nd&z{Pw#SvBW=
z6m!|?aWgbU2QqFjJPo-~4`nEA^P=p)kP}|$Iz5C^x)Jt=)nPvjsz%{zD2R8k3)(Y=
z&}16JmP0Wb6(=A<7l1UJNZ^pjDE8-N^@ci{U}SLd2rwsDzWVNQ2+df9AQcICKvb{-
zmXbUsr_vHdrt5ex{M8WBox6a6qVdpv?U4!1ip>me(STU{_(7?UEKgw7wZju&+Qh=j
z7o`u7-D7ia6f`IRdjo1;xpd#-Pk?67iYQij0D(b&IrO1jb@Lwz2?8KTG!V|TvcKM}
z1u$VUBQ6z*{N{My-M+4g@q<L%8L3Re;~r7d<ni!j$n<UnHCDlkZ={f$KDS~My<ptn
z4T_gS&_CP;O*n-7_=!+U1)65wI6DN#`2l)R_3j3GcR9ebP14wK^cL$8&@5MA7M>Of
zKqoX3>MOkjD34)N(7s&MH4J=rIcOW|n?i%j1^B9WweJC6@IoDE>h;xbfH>CErdsji
z_)h>3umVux>(Ka`WBI@TGhculZETG+-N=UM2ni5z*@g^tiV5A75bzi|x@f8ojr#WN
zh&tx1NHHGY0(WRmv^ba%lmK-<yt1mfbv`;)OolA%ZXV-CpfK|?9?B9gH^SaSA>yXo
zC4@*bc#;nx_CMF_d6R%qxiRpbZq^M~&~>wS#4SA&0?vb{ZZ{Iw1n|)0qM9UZ9qfZU
z@Iei@T-RwsCp=7E(hq_B?h)M%-h|mfJ?nAg6jH+%ou0H_Gvkh9^MnaU^P!ZsEAf3U
zt&edKgS`#f!iQhRbTM{>mfqDglkX636S9I7V*_^;hs`^0rrvAfoI(P})0ctPY-oJ;
zY^TQo8EOj2f>dD9bHE;=C<D54Yp0kN2cper>=Ba&m_hD;@KAgg588)TfTR9}ODxLM
z8%()o?QdlJgBfXncr>5rUduqexA!q}U%cF6K|abw%U>kAGNvWb{^U55f*ef1$I+^q
z9h$fbcl0*8Tn=l5MM4vvH{<X3ARz$yLIDC-JI%K8>6C6Q=0}VXU@;ueuWBMgasK+!
zyN9WCtx%30YPz89LqD!oS{mkIHJuzk2W0@FpDbyCLcp;3slG|sPZ00TfcQ;y7m9$7
zY03v{)gV#J%Z>9)lsWE>w-p&fT}OpbswNxT&w{ZCX`heb$FPo^f~AF8;1!Dp(1<FB
zY2j@HgUN637y^V%hm0S?wcSdq-%s1ggEpMo3}}e^!mZg>LH2wDYBwI>Zy%h&{jyM*
zGSCPXM}Zt1noT1Fa2*4oTj`+BI|DV=6mo_jQA3gHfFhKFx`r=F*ZkI2oiZT*yxDkR
z)3aNP<Ljrde@FaTjvin+Qc}JFPKb{Yh5<+jz`?Fe2ycK8?OqokM2~=;|7M<90>laF
zd)$sV8p$39QGwkq@~rO>lr7iffD1?GQT7h1sD`-`B!5B4bxl;zWqP4f<xhb=)x-(1
zs2?fv*srq8KO5dW8|<On4^Z3!D&hCx5cH_O<SX<$%CxX+AaOCl6M#P?Ph;jJpAD4{
z`(rYYHq<@I;xjd<tOLc6KDkvjhN*7u3m^MneFgd2u=Ng!fM+QjU<6Vd)XGa?s;|Ia
z(tP45FLsaB=#7p4RApJl#tKM<7Yxd77VG#eX#HBRdUbT;aw~9*l}~&{0E?R9<2q<^
zxefz<i=nbmwg}S{y|ofC3Gn<MBh0C}How{Y_ClGQ9@~C7tYh}q`M>@w(Iv1%@BX@m
zF$YwYJlYc_P$n4>4`tuyQI0;-S7)HT<8qIH=#$$&Kc71qgodyOr<X|3dXaLZbQT~2
zcMs2{kJ>`gB&~_^if2e9-P?*w54s$mNBe=!-&1G2>OPb+g~VLvf2iYj01XBs&Z``g
zHrOpx;wTxV>ZeD?{J3~kDvzC8ay8%vu?a-3o}U!#e&mN+{j~ODog%XDvS}C7uCD0R
z4%G37%g^k5Kc}dxFbHz@2PV9$QAY7}-3td$-N6eR-hA9RF%F+CzxgKd%^A<L@ntjD
zFsivLW&<rvKu&n7so_VNCFAMQuQcxTIz^5Z4pV(597-}dY{C<U7hKn2cA+l|@la-#
z64R6-<v|0Ma!;1!ALtYf@Ge_^K;8_dpc7F!LurN5F@4Z%)Y_)?OExrY3>0*}oUniu
zTRf(MH7o;JOVQM${BM?2B2$Jo8fL(qcPwBi?YQPbB@pnL@O}6Lm3D)^DMhZE^w5jA
zVGryr<-4D<`$DaUQrQr))!D{!TkCfjx+yB%eytbB`sPFGg1)#KpbpT3xJ`PbFHu2d
zZlQ}53uRCa_$C(0K636vF?298s9d3RPQS;(S_DVD4UVD1t7Row{|=nzN5umAbS^TP
znEKCw6)#X?)SP)1$v<byq}PoF;Gbn9CK*iA&&{p=3#SYeFn8`;nd)oGHHx=YMK0vF
z3z;Gc%*Cc1njrT`snQ>6!vwU7D49_Yi<{M`f=<#UVA_Is9XYt}&(n)xQ*Q>&A^|N$
zGkO3kL67Cpf~}9l@S&R2!IFW{*mbZri;$05S{+s@6tJY`(`~MThZW(P0WtjgEJ75&
z`m_f;HZLn?#i(5D?efTeP*sc8$BrVVqc=rYDpq4AG1IzI=2wMZd_XRK{5bvW$tuj-
zl|n2U$`QPDYob-31#CG?VZyCZ>mVL+nb*EvE{%7qyAT)o7Aw((|K@OV%5d6B)Z4(M
z^O=v>0NzNZYF2%-fvNUuc{svjKvy9v8sn^51#lL63WuQ0>RQ9zz9Bm)$q$B6be12x
z*)0N%u<OjaCuwf4hFQN1;$c|V+_@e%XpzxWwAOhk1=h!&Wa%)#fYpXk7jU9d5FUyr
z)pop|zsW7WT0U8i0SP6)0`uVI%HhV~%Z3etB0(w|KcMIU%v}Mmi?KpC`v;y^$nJzx
z^a#<%B2HT0)c4^7H-<&b&mgzR^;x!ZZJWO<ET>XE4}j9E(2Dg&DaC@-BniT-Ys`Ce
zAu%tb`Hc;@6dV&f&q7^wJ7DwD^G8-bSomZz!*!wAb8egCYnpKsFN$EKAGUus3nGry
zqHqWzB*{vIv_O3lzECiJ?v6>1o{sO2%I=b|9!JENkB)CAl^c$&YD6B!yDF+9XPo^0
zM3=?TJ>k66#2@+i8urQ0ExvxrRK73Bpm1qN!ZQ%)ES@$T`$xHx^4diB?C)!eHEp{l
zD_;7Wy_cF6u3dP&I?KXnG%{Asg=ruN#y3?`h;{Wb<;1wlkE_~X;xMH_34{8$VI4~E
z&Mv<h-h4{YQ+kwtWA=UvWSk>YT3X~IofB}f!*}OT;4z3-wgy6!bIii_fimTTy19pb
zNJ>qoD6iY8!!PMG3xc^R2{yjUDX<M#NzAo(3NKn3SA;oR33zLloBtr#JA(4)T1|R#
zFbj<OdvTrAo|i);l+=8`Ht9;>7ETw|c=YJ6rBod+1HLhcPhzgJv9-RDzFkD*=|@B&
zFAxVhiNXwOsOjO#t=+K(wyJS5Zci(yeC6Q4<Q8T>%-F)rXsBN#DAGdd^|k%_xpGcs
z;ZX;ZDng$@=$ni>;^gfawi^W@8>Mi6Al=p583`h5{6=3K9$OzNe0WsR(((-JYDBl$
z9&ksYlx)-A;40KvHCrA0&te15zfs5(U@LGl1EFs<`ANm7>B`DtuGsMQ;apb?(K18a
z?asOr*+>>Ng>Y!%d*WiQ<S@xn;blchjE95&xCTyJq?sFaOvs4TKKNhK1J$Ic|Hunm
zDdXs;hG;qeYzn33sCz)RU@oo<Gs^OsKh9vLIYc=?!^d2FQej|>K;#&Q@+k9tOVe$}
zZ^|3iAOo_*7O<g?*$i6pr32JDkSOywMg0E^)X*GO%L4sI+sH#7rAsk@`e=S(1jMN`
zpdUhWoJe6o=5Q1x>kr_Ds?0oKu$=j^E)|g5jxWXb#Jb#T`G51NowvU;bYLVkC<V(o
zIr6j?3==(u9&$y|PZ`X`{)tiGgP_I_cnA%@xc`M*K*`nra`)bb%^feL9APM?ffA?t
z{?!qR0GWs%WJlF$(M;`v$b8B1-^k}JwCz5b0`rGHCp`XK4q6Kxa(R$zQHQ{e8rKpa
z`2gI`akpax#5)Hj?}vdn=9wO-2e^Q0{D$Yh*hI=m1UuLp*E9?OBcKf`Aglp5nIfF5
zX9Rx=)xXz(fUOVS`f=(?rWulK$m7vA-eF4ywg1J8)4$On%gc7-gu`;j{F(C%xDS``
zOlCgL$&lZiMtY=kT=GkcHz;u5fPe!2`hPZu@ULNWVk*2)yfX7RW3N{NyABPUfp`D{
zsHb|Bkl}Zj5(o|TX*MUQdN~i}qlt;eF82;ZCAELaErZ~&aD}ea>|hz3R#ojy5NJ|h
z&7MZ!2{zbyAH=wlyzhwBzz1Y!+JPRowyHVYTU_*H(nYcoL=*Q@m`uDD`eJAT#kffM
z(GVlZE1f(m2Q^B{bmrBM0ePsO?ykV@BMe3Zk*ZAOAR#7^q22f~PJH0u_}VBEnibmV
z{0C;j2N5J+(2<tT4110^s}BLm%kiu~$Y~4+1ChNUF03a@!6m)}6GAJY?tm3Es*^_Z
z1RH2O9nZx?4~KsQa?(Op_uhxcvr@0xe^pKf*>T>vM~7;3Z-2eR9IT}PVcqd~uQTYo
zpwrdV2GwmDP29S~p|Puy{m<U-18uJ;wb0kw?fxK~69hTe%$rQpm&sPGfU5YYtqfdI
z|IL%x8Q%$@wXk?6`fv6$pqqtq=Auo|TrhbF=o2NNMAfI*BmzpxC6H5G@hgaK-vR4n
zHb}!+C4E$n48f$VwxjW}<j~P4<p<+EoGL%GLd4^5yok!Sd6OLU;(lNRpTPYrq!;(+
zpC5k+U81tuY5ZB#_?K1dYAx)g7fQc*an9ANaI|EJT)RE1w@lcE#o&v#4F?PdDz?gM
z7meo(tF=?53hjqVMC|(WrDmf9aVITtGHcT>jGFgAA+^khyuIRRIhJoAA^%Q>5A;i}
z5gzcxwY(5F=;kA_@)w6$+y8jO#b>cfF)a&$-;HD06|^{c?&s+jIk2n1BCJ~D#hin<
zP(2^+O;@=uzWnW$W;XE6T4!-p%(A&cu&|;*J4104;2e2tyf#ZpStMRoM6pOz71bRM
zFNChD*i06nl>BR{94Q!2H&$Vg>%+BSxn7$K$)EUdJ#UJ*%r_?Iv1GGpSmBgKwGPJH
zbq`-zW};6AbRgfsFHZ#rip-Sc{leetWGL1g%<$9hJpKHrsUeWAjq>;T%K{ZZn>_++
zin=`AEQVz}(=2IH<yPo2)9PW~yfsu}<r;X7w!DOHj)DI{=}Ly*-e)$=(`PSyt>$bD
zq50-3`@l*Z%%P9F2ET-No@$aX9sG*mEANf8rf>|oz%l~zW?L>g`@nNdVnNa*u*rEw
zGKoq#2jl2zdDCxMU@6x*d%D05EERAW|EnBMuxBIQIl<HU`E*;@^|e$5?{dLsUo#g+
ztI?w=xJ3;EO7VB}P#SIgq7u12yTy6sAPmQ{a_v-%7k#5iwi*?+p!w=*4o^jcFZU%r
zF49sowV;T_4!yIpxWkAH+I>-PozHayF>Mh>&DWf48BooFD6A2C5S1<+(*>9|W3{d@
z1Zf2AXjYxMX3pMA6T1a^-*iLzf1Y#$^rTH;(~@d`?`(z*USyL?LcCPqUI)!a=gWe6
zrK^;f0N1&WvU7DnO)WtE{?o(vA@y)KOl^?i1rEg#3ej=jG_f&didZvsZ+gwZvos|x
z)t)O3=m{zsXb_`Gj#jdkMzJWFhE=AsTov<J2{tIigEc}sz|s4knb;TCX4*3Jita{V
zG2nPyC@Md~&uxC`5`IHz>YelZ43fBOaqy3sso-kHFdIi>=Hi%0(U>BY;PaKAALklg
zLD@7&%dlg_9J;U)xZt|)ji0WfY?q(?cmX{;&0${DGzCd3>`WHjt@9_(Q$B7GH7vKc
z&F=HIwcQ^svl|7@04ov>5nDO)AWfV*?7>itH#+$Kn{BY~nwwg|{!u1HWo|8E5QkuD
zRsrp*#-&ze1?J9%VCrhnfac^zRo+=?l5jlD<%5m17CE&FlX!zr<kwDlv6&4Ib8k3d
zb8e<YX|eIoL;D%a2Oloo3g5B18bjUuw)p!x=6QX<k1gXT5P5*He6mTR=ni%;!!kCK
z*AZLJ&x?{hyI-Ag4RyQ_Bk|C8IVo%1!I~)R`jR0XK}9;x{fWfdktW*{M1{tY&UU0o
zwy~EtevA71ZfCjOKnI0*K(H_JIbB!ub(QN(Hy-`EMT*kk%jda{8uFj-%winBUkT==
zlsD;k5{LUt0F$*4yhr7Z#THFVTw<aqDgpu&i+HAzU0%zr-%FD~ui7$==?;f|;$H!U
zz=2^VBCUm*8+cMt@=OXrV|0WE^B0SMyjPDc-;i@(7#yXVN)Crx?}sBVhQN11PtfB(
z-cKk2UyaofyobdrK_}_AUek9piqZXr4}$B4fOoCsMac*4sa>JlZVynfA<!kHI&I<6
z5JdIvu4QFSj6Tz3fHkdHqblOVbON7ofXvJ8dpygR9|o!KffFw}$cEE}ba&0^*-iVv
zuji-U51kX07`sDp*}3&;o}Q#A?^H5zU)Apd1^YdVOoO`=o(sJ#TU;D_|44i7cohpf
zPGw%yOcaqkIj1l(sYu^-PXoq-sGKnW$t=%KH`4(+F7?dRBIR`DLGBPtf$>zSRU0FL
z>EzOMJbi<G8)+i?oMAW`N69kvTJ1vd1C3DWn3JtDI00xYO%}3^Lnvy}u>~1%ijZ2l
z?E~2ju4pyVx7Nl8#m!EK8mvr*`(8P%Qd?h=U0|8K`Af6%0tND+NYU46uDS^iKc%tV
z#h$<6^#ih5UJox44%mFt{{h$M{XUi|vc5utM7w4lX3-o$%@(DzA~x&2y6GZxZu^?(
zk;B|Xx&8F|I%9|Ce}B<^F6xTV1O_d9_gWFYn2V%mdq6F!Zo~v1@^rc>JnS*Ku_{$-
zTJPSqcIjSRJmMX6^;+*rw5ANg*2A#hj-S!zHo!WSiPeC$9e-GL{2}F3GRNP49$~LJ
zERUBYghdAEcau6Z{5dM1Q16ixM(!E(YbcIJ=8u*Ei7ANg21myt#zKbi;08%ZEzQYJ
z9s0x1v8%F5R{$Xq`X3c6_*ee~tzz6HAt=P<Qj5}nQA3|m()0X3|7%z(T+Z=za7?=T
z-5PuVE=6Xr+OvoN-uUA`r%xMioZGhnznh*PtOR$kpO@=E|I7y;Mq$Gjr=<VAOd*33
z?8SX>Xh7G;efg{7xFQ;yuSpxZ;7XkX5VI4Bf9{%`{n6n8xWF_~-aoIfrU@T(Thyiu
zj+c6`)g}DYt$EE&>os8gJw9_6#KY&lO#u0X6E2p+Q$2=N>{(2%^VF2`fBcBEiY3@8
zXTV5xs%<b$vfA%puhO<Br;<oO8yO<V@f4dYJ-M;oz+f{n*%*q~U?TDS$8VQWej;Vl
z`Q7TroBB~8lwSs=2Uq0UFXx%o1XmCt6o)+G7*__%hc+izp4afoVgT~1Vg;+PLRu(}
z_Ed)H{JA!4<XEk*w|=z;+;eHLxM%CKb?ARAjA?A-960W(z&2p`Udn!O;{H3K-yr(t
zqMmy`+L|Cp#EX^N4^4qH<^|d7t!6Ig7muYubK7$lS;JE9%22Y^I*iKM@pMiu4iur`
z2Wucsc!lijs`en39K60I`p;{k#@u>thF?T)%Rs_5Bly{_z9O@6ti%k@r$%2+3-$lJ
zQBzH>;0Lu}-(r|vZZ8e>JpH6wPaLW283ibX+31c$(G0;yAHuFG`C-C!7luIyl&IwI
zr=NNIq&_%trNKta@OknUb_eYCqd`;*#-ypH(f$G>?aoIR3HHOKEkS$0nl$qT1VWoC
ze~p9My9jt-BKn2Kg+9A$QYR?g_QLl13XBFW3mf(-(vK*3Z8#Z}yl@7cf#_;LFEc4H
zEHjSd)6Ulp`T#2R)O=r#S`{B?|Ho@3`v8OR`}I~m|1^E_tyM5e)u}yI8XYoS*%TL<
z$n@5v+}?oTxH?hKXHVi?&zOhi4~LuEPkeu!`w@V~9xsFXl~by;qZGZOUB3{+V%Rib
z7h$#2<@&Z%mmq);-v>~la?md@ltg<F4c8xh^8uo?l%0?3^pnyfPJw;Sc$9*}+OHN7
zuwUzm!bT6s+nzvD^Vx2=^z)21<+)uF$1>>xY9|hhl~7`Dj<CFL6Lm*r(I1=|6a%5r
zGW#}*e`2WgntL!D6*n_pXv_$^0`j2+huYnjiSnKf>)g}kckM4RD)VDN&8<*3OJ(*g
zySzt!(|w+#T}L;LSIQ_J^_bF9(;BbzcOe7JcyZb_`p3}-htaAsZ=j#hdch=OJa_<#
zgN*F|NLWAcM$pqr{!Eu2ppZQX^1M2^4*;Kp?>ty}&t?0m6gUg#LWc_o@8E3Nspb0#
z2dTpv8ZcK_yuj1ekJ_pI(JT^t3tlma6JE=AA;_D<NQLv;&HWRVf%RJ>laxi)-pi){
zMi%UGh!_S@)cS1uqMY0O^DzSz4rrGMh{NAc5U2$HuzoestV=--bRNq56W}ZIxZNXz
zYtbJ3N80x3mnY#Lps*Md1=e>J)i8KZtHC>+e47tz(HeiF(Cv3OU;cGzqjG!3h1U}~
zJ@ka3Z`Pg?)*Yo@)5zwwpPsE4@Z^5BX|dG8K}aEnmxE`SO|#r`<^xF41_#Q-dvScx
zAh`5=anKxn)nxW(GF*X4tE=B1^Z|1V@bNc#SQf(@w-?D<#H$9P$4AuCh}M+s1E0ZW
zkk;C;g;6#|Mn0dBCv{Mo$uVBNW%H)#CJ(AIIz~DXMwvGOXTx6W5i4!XrLU^LD*7XI
z23QH}DP)~nG3>OCH&3PAq^G#ab@f;5w)}jPnlS?z^QuoEk7J7E^#c?lnG~`9j&7k{
z?ra1d!wPaMa9@AR@f8mjDazRyi}-4SP{3)cEV9IAI-h*k=_T^6;eT914D^wqQu}=8
zyMazqjJhY26sAuh*yi>Z?S!)?JAF{*9ZsH1`Y#9;5;s0ctg$;xzwmKx%*?%|(mE?0
zjdQ3e@Q{-9od>^~>(OGx0@!1vXa1o@6X^%sV1Ehf{iI>O`F-t$_d=4G<y}Z4mONJl
zjsixVXlu4<bG2br#1FF}+~;uYzZ+gjyfsJ;0utZpcXQpM&#J$5rRB6?nCV#9!%7z`
zZ=*AlR}tx?i#$#wHJ(pY_0MX&%4S*uUxNWlJy)b1>}}oHfA8O*4oEqvDg@#87%!Gs
zGy(2Tht<s=8Pe7#P`0fbW78>;cEf&(T0wX)*}7pW=SCv+DV2v|)6J>vs*#jF>>^Rt
ztJsYH*fe&9I57b+)J5mUbvd$?r%5Awx88+6oTZ`i8>*X=HB7-L!mmpV>aEef)2>{?
z9meB5SAX{g`UH?CUN?UuqG{*aw*1|XasVso_}zuhEFVVfBX+_Lv4L=|r-3)&-Jr`O
zq#B4B$U(J{J8%C42}174;P{)P2c9J;3V4)w4@T|G{y%*PwuuwRAHVqAh+sdC8z}sH
zIs#K*NjJbj8x~JHrGFeED4n3V#cuQG@RcblGOiTo`#%vQr18`p`E?%5jr*^yjN?b{
zQaU>~RQ_amz@YL$3yOI(=~_aMRy@&mD<{$BWJUC{|90gj|L*uGd6eU5cQnWi!NH}r
z5W3F9j{|#Ub-KLg@ICi|LX)i(tNejo?XdW0?7Er8EBl_N)-3;w{mCW8iQ1&?y5;b$
zt0wR37mW{IFB&&0FOCHANLAD-Ik?)PiLRj!BId{6FZpM2(<al+o#k%=uXQ!ZR$JPW
zC2;K^;y+kzq}`JBTsB6#%-lyij@P~*r(e@oR0$h=i#QYwy~ySQj-_y?omF$Y=ZI=o
zuKpvf*aRTZoGz6+cW=3hAE2TtRs}>yzv2|!9`TNEw#eyQcRChMb_y4ESsN{V8=Dv(
zG07<O)DF#ejq}?}gmZLEm+q@f|5!7V^;4F)8@G-X!8TFwV}Xmfy&WVY!baskALQ(#
zV%g+h&Lx4cq&k?IsS1>Tx;k8o({3kI$FM6bJV=wFZ{bfu8P6p=Ti_0o3C6D1KaOh#
zwZl13qf-S_Za4<#28%66LDFx*Fkaao3|l=`-1ZNV+zeIT;6TgVYLf^OXfoLRO~z||
ziSmogAL{P^%~ig&H00X6*Zbq>fb6;5<LGVtEo*U9;j|R$ub)N?9}e%)o8%r9EvFs<
zJLv7u^)(*=;(u!i9ZN1Cs&h+~f|pQNtd`F)v`))Z0lf(Pl19w*ld#A$qG{GYlg!tB
zESgscj){XS7qURX;^j8j)~mr^!`FaA+(X1%VVw+Zy3@_kcx0{m-_yc6B3vSBuhge@
zCvJ3or-)bZ_J9y86<}&a`(=SAKS1Qwwz)WfzjAd1WyM!C;kD3Ka>P68d2tNe^<KRi
zhFlx`wZJdiJtR^goH4ofj0weT;0GkDBS0WK3COb|i6SO)`V~&*P)KHkYdSHr=XI7G
zXJz)_tI*m72M)NBk}cUCp>{Wh_4jZ&{Ob0}x80ok7Wmr0<(A@9W8xbp2bqlz&)7@)
zB$>Uc{%)rB9q)8^AePrWw9NcH{^e<26)<cz_kDhh8wFbIEw&1osRqFcc&R0AfaU!v
z4)kjAJXLJj_!g*$ID<K%|D<aSHOJZ_jJU8FtJU9M-$tCBD|%*su(!$B|Mt!E8EITg
zxyD=WLXi3{gK}Ak>=clXxB!1iL_3Yr(f&7KhKo~kvlEV^c$UeT30TKAs6Y%>(q^Fs
zWw|QGG7=kH-_kLi-e>1U=|s$(rMXh!K^z`}i(P4IWMh-}a08LB`{UdP(TGf283snS
zQt)Y*&7=n_>J>X3h`z#&zCM3Fqb$eGs3R4S*3zOQ1Jh{mSf9hA4{lfz%Hi+oic&c^
z`-Axnx_#0km_!}Ms21>Y!$G?Ij%TBY2hR%(zBQjs#(&gIG_2Kn6R+Sn220g2P)`oK
zu5_=^u&j0X06^L94%b-XE<)!=W%iv&!)k+<M#E{td`}xhbBacOFCV^0w`1P8Njo&P
zOtwf}yy;Nw{TvI}->Q_8I}+)1F|_`J3m5q(d<!3ajPM3DKw`O&Z!2jWicfDm=0n?b
zWrs26n6X@K{=1mOUpn!uz_e!AfqG@MdLSlurY(^m)?Beedal6lpw$0Mxzfjlb*kM0
zx!7M&&W001t`atI#uYRw#y?jHztGLB%y}g5?RDE7wFQ4mnY%zyD~ARVxa*UfMgBlQ
zDx#foi*c{myzy)zkNU+n+)<${ZfE>+21AAX{G7Aj`u^5XqHmJm-6&@Bb(*;<@a7NJ
z{*<>@9V>0e)8Cf;Nt<Sw<k&FB$Ms0wH1!_rNFT<t_e_2IQzK9`ys#aMY$I-tRJvB5
zI&(gW9|`J=QdCpdu<LCFl4>sf`3CK$e^LZy(nMJE2?Z^-OOG$P%>K09jlc1@nVmOT
z3Z7D68Sp;t^Huh9m{ai_h4MEkO<`m&KXAvEfoDQ8_ROvVP*3m$43-;Lx_H>J$KC~t
zupm9G6g;9a43W~j97k@!Y8(Io1DodiQBZetZw#gNgfg!TpSAF6V1_1g2W~kF{u7R(
zyNnU|eDC(UM>J~OK~dQF^yL1|4$+}6MrovLnt!DHKz(#k!ZV#OlccC5;}b~EFd@IM
zZSIJ)-b=RHyHK185{qq6`M)DFze?$v{o;bmSr*5bYt0E_cHJMsq4K)PJ0SxtpY4Ud
zwtiRjkB>gy=ZhFZb<(X(3_jPo5g4i1ybc=i5;`(C`rX9q4;!Go;=x3+gmrl}TQ8|-
z-}k(8Rv-dPZd;N1Z`HkLSW|2FHz<M<4gv|Nfb_0MI!Fz@D@8>GrGtVfRRn@`3<A<b
zlp+#P5$u2>HS|!F5>ylfq=YUt^zN+9IsbW{cdl!`%*=;(&L>ZlknFwhd#ztvH}A0G
zCvL7CH5qz-%d%(D1<O9S2Rv%RXhg%(uUFz4kDRzRQhsxiRhh->t23{FF+LdSjjIFO
z&j-P{+|(4as-X2gFr-}b*#Xzgb)VWCqhvRNeduTV1PamcQcH@2JHfun!Q+b$E5%WV
zvTlEqdLT%w@y)rHXMfngTf#KpkKJphyW>R?QVrR{E$-*PNgbqo(epwMt5SFwKy5v*
zws(JG9UPkBn9OG}hdP)--4eh>+d0==PB)tUKvq|M>!DQ#)X~-#13q7v!lYqxQ0C?E
z`7~*Vh>wSotlwWY$|&M=D|Xncds;4im|;kD#-Sk3EX!lYIdowbkS~KQNqw{u;`yRG
z%J(M3hz~l`Pd~Kp);c*}W5ml-_T{_Uu{)L09@3XzXOv9-i5S-@*&aNzsJV$|<`o=~
z;3j^SyfmS2iKX5C0R*!@#QO~-+hv_wuR4WU@?voX5<VGeR2{mmHy6K2wsXuO47X&v
zQZeqg2GXua==P7OUhUxDR1a9VoT?sttCxOQ@TFtJU4?ht8Ze1d<BfP@FmP`v`B~z1
zGiZt<-sg}2*JHjl8<JDBsbg9_!7Ahr6w<dgs6d001=x_R)=?#g2uc|MpjmLM$49&T
z@Ojb}5EYO<urdAfgS(yNQ)P`LJ{Bd}yv2J$<81Axl#Facw|z70s{ZT*s&>ibIZ0e9
zDu{I@kO#RB&9cEtWF^zHv6HPMg^XXC0h@0Pe4pF<`udaBGHmQ6U(hE=1k7ix5EL)5
zbGk)X0r+J$auJi%9>=Kdvb*Dc+?l6rQQr7-@kdUGv+#=QM88i=Jj2L#nB4i$ok7Uq
z>0{X|cLNT-V3{lmr)QK2y`B|=p@|HmHKhXy<vBw>0JpNQ&rfBDrTfXf6w0M~IAJI5
zL=o8bT%p}7Wlet29!^XDs~-BFTelSrNIt{KZofj8I@9N?{ifF#7~znL;A<5U?`mt3
zDeIw*>||~om_-q1bDCNiyt(quq5e*ahEq!^hA%!fhT7yaGPl1V2%<Y_YYj>}v&2Qy
z7IGfjSgm)fcWgmpMWg_s>>fpFNR|Df`kSa^VSl3_4)anTZR@zTb!M8+bE3S%CCwqI
zPJ6;ul~=QuDZj2DcGX-|Y^VS?X6k5(UCbDLdIZzHPjOs1Wb^Svx)m|%k?MoT)oDVF
zahy&*ShA+{xZG;0gw+o<=(aAj!vem(w`brG<7y20B~Q%wZH8DEQH)2ZkZ79l{NKY2
zHOFI9^Dn%;gwO|L=g00omS*^~Bp-q6d~wFFCAQmH_b&=3&3hcxkxd-8H%)lbSX4q(
zizMmo6KzA1;=PQG9ASS~A_n9xb74K*$$-wpDC2qTuxoZOj=-QuWfNQ0#F|SkJE%Eu
z7acOwg*N9+Rn;#*-1{=@F>BHCBssA?8Xi<`j$<**&lJkbZ!2tb-uFJsZ)Uq4I3G0e
zV{59QL30uPWEH!}dv!-s<N|fNL8ipvV&dAUWp;J2N{K1F<uwi61Ubnf4%No+Sf2jJ
zwbU9*uLEAj1rpki8D+7pcv|lyybBlO-Er_*wvbduljxfy1v6u1Vq64FYabi*Nk=>w
zFv98^-zbXOpPs7xlX6TnbKs-dvjZCBI}LIK7i~ErZY(DrH<ln6;5F2ufO}Y0|Fb$Y
zG()JHk7^v!Dx;)}Y8d8j3Nk|pj^BrvTia*g3%#V;Xeo|*(1B%9IKAU!1niA3p<<zf
z{GP&K#@`q_qlP_Q`&Lm~x%7Gbo+}Hr^XJ&wYeq)u6;PAe_30fz)6b8|1`Gw{Ze9ES
z>ZgR<XCa9~qrZ<5WcJ6zBw8(WM#gYDn3YDdll$)G?M%0X3FXd93w3tyPZV|F!X1y!
z)H}USe_@X&iW>apnKhAhWOQf{6?p2b3Nd$fo73vouWiLG&SJRKCEPn2qS-P2qWEXS
zbj0>hb!kig4$l>Xzqi-~LPvSp*{ML1NcDDqDdAwZf^<@Pq=kK^itmjR*E*x)ST?=g
zyxhJHM7!{%+{TG^MoTkE5ip-@D*Qf5vm`K4xXYAAEJsGT+7{z*V@f&VaLHMtl10g-
zu=AX6=Nw`?*7BI%`>B^L#XpW?EO{B1M*U0ZZoLc{@3`iJX79}L8Na{&Xh7Nr#hi&~
zO5DODO+lL7`<b~`L_AZBo*9qPYHNgTW5_%zH#@NWl9z-DECg1vb-a)%&OWjC3d)Z7
z7JE>OzL3eg6;kFM2dQ<q8yHfMLFs(WKABgjgWAfd;dzVSIM;MkS#YB+o;@i_Can)O
zFsKj+G|O9^^fx%Pl+K+4hf99*EircW>Kw79(>6|9n?LLm$o!>+WE_mCBb_l<V2M4z
ztt3rpv)_9;oiXzF_t#cjxEJ?&{YFOBMvaScIaK*?9@rlbqR>dxM?&<l*<TL)(I*_A
zv)yWz5vi+qH2$Z&xBpjP&CbD5Ga8d<{bBO<sz#*Tyush~W83-L5B-(Q5`0L1iwRNw
zDr=_9h{W-U8A`Tth<VK1zzV!q{vwd8d~CsqG}X|+(exI!D6_Z{b~35kM~hVWS&x_f
zY|yv|W7D1LdpgoS?BT;Bz|u)On5k9b#@5V!JDNtvGKKK<sf`FvMqWSL=y|~N1X)Bk
zV_;pvCt+yyQ>2uSvd`cB_H!os)3Rnhj*Yigrd?Z<Oi`7?%mR!1pAzO7$J_~7wML60
z$!3>Oe2esp&5)3DWDzmvYY(svWBXgbzwhEGjnAy5;~RsV>%3_8<4_OGVhl*A{)?K{
zH!6;hrF@?hhNyZ33W!rXWy=ORM@~M*nbV0!JY;~yB9n{a6g67Wc|05-szE1XF^jr7
zICD7X%S)80sGhb&UCH9Jp$3evL!Af<F`+YRh5uIB@||q4Oe?RUsPz{@$r2F^0&Vs_
z*IB~jqF9p>^O8TYdu|js)C=}s5w(J&9g4$C9=Qjk34b(s$DLZ4>sHuB-nZ|Db|+`)
zmydi&^BD@wiHgZ<PKj&>*`-K1K5@7?wI(`$#b~XuZk@p!#xfq(1%m=(b%{G;X~UGr
zw40&+;is7zf5bX|riA~P{Z-MsPtd7B$%iY|pYUM-h~#UYiBEzS>>URk6NZIP*c@a?
z;$Bnq;D72QLlAQaF`2M#x)t(gi@A(LU)sl3fG#$8U1HImZ^%MPX|3&stS8WrZJ5?t
zBFm#Pnnao9#c{Ea&1f&viLI0}Ll>)+;aRqE?)HYQahfS5G0JI5t>M?`VCPvw_r(fy
zp&)h3F3l3nV=Sod*TtAP=w~dS<HHzdCdwizga6C%;da^m@nVk5k=~)j)%AN)lWyZB
z%U7s8EW#yi6N#4izXIro$GV%k>gVRZ)%Tv2)Xs`|82mzx?U;5#;e3R8-|XN{XL|6b
zt`ea>|Ccm$^C_8|zu2BBriZC7m-M;ZbuM+O*}r-Ffe6=1sm>qMH5@Zv;1XBUxzayd
zM(v4O^Q9;Nap`k4-S7Ftv`ESW!IugohvsU7ZntPho3Ak%&texeCH;e+a*iJ&NjZdX
zF|(<5B{f>N?32|_Pd>3&=s%&UJ$`7+NyC$V(NS%A$tT=war{7arADaCz-hA;CIUHP
zF&53v#Y(0*`i;%>l$$D^nT4K`<oxxi)BVpvk5lb(oyy!36*sK&r|Sd6=&n#=a*cWA
z>yyJ>_L{jAY<X4G({+gznOqC&9)BygZ|Svpu_cOLrh{~1@i24$WNTpdy3j*YCxx;<
zC9a9vWha#qrHz)|iM=9vH%0{TN<5~~k7<>{1rw;+K1p}^M@gUgQqZron$MlBQ%3cQ
z+^maFF>tJ&%lZ2BZEUe^bS!5GjOZRmWokC>>&N?49?AP<{EqZ-=V9_;TbMI|@n_r#
z?i`vS9otz{b@J_K+q{yhrrT_#)D!x#R&DFM{_5wFj9;tAcSil@RK3n#%__@B3ocW}
z-5l@7nCt5f$BX9Nb}`)2Sld4DQ-Ky-Pr{jsZq2-(-}pcyz;)_{{tX>2bSs)%W7|59
z9tz#ruGkL$Yc71hw(7isLEwiwXbL4vSst6HZzYB|4ETSdj{Qe#bD?Q+!Qxku(|>e7
zgH#o*G}JvZcLRteg78-9T9XV9qRi?3)}&m_;ZTB8YzEuuI~~5|nye%`6w3OH86z|l
z&o|`X{H;k@ub$7p^=0RK;vGU_^~>BhAPbcbHzig8sc-4h_{cPmPd!NKo%&OrrUaf3
zTg|DX^9t-oisZWWcQ%4ReenLc86+LJ;)XaJ{3)au;o5F`8VQyn5bTl`l5pw_OzBGA
zPcy)3;wt!X?t^pN4X36AlM9709Cm>7r+PlV1`M;=2XgiJk9<2m%v=sROGD$D3Qzm0
zfX*FDjRx60BPBN{&+cM%U~MY8>_UHSaMJMhuV_00apwUF-b$uI5RA}!2d)%RYbV#$
z^*uCX?>Z`H;!wC~+;BejT_o~pwhXaZbeI5eH<bb9ke;eoS_L!&Vy{y0l$d(KE_k@E
z-%+JLPoz^Ox(4C<A1nJ@N(Z+a9zlOF2F_+YMas@&bZrSfvFd$W;B!tj`D}r)qypOf
z-WzA15)|Ti)q1Xcy59i$!Ka1S7v4T<NfL_UQoeCnSM!_X!qeHH{hL`=dUOKoXPhg|
zR^3?=Z5+2*CA&PU{pV~gszZX=GW*RFR#zb7T}1m}`{@kVsZ8=dDZ)DM>!I)z++nz7
zXX%k*z|aawk0v!3AONR|<`Hhe*chmjgIA>q9eS~cFVYXu=%op~n79Xgi>^w7)slnO
z_&&QBRuMsMo^F=#9r(_dlpA8%{lOxU9W2K4<^1!LhfX!+nq|%27;m4s@W#?xpHu6_
z+#Mz^>vz^=yBZiloeuA36TJLAWwt?NeaO||!H4N4`|q#L{~*A5stwG0=JttY7GC<`
z9IvJsRT4B*7x}|rrzSgu+kRRRIGEmc#)~WCs-p-ex(!y1q#a=?NpyecpJ0T|=uHWs
zR1BobcBiUrR@tj<ZmO6oeqNcmY`4M~RY$tJD=>R|3sHll6{vAd?1g<V&Inl)*lYRE
zeC#R*2ItW|M?SpBUXC-dU8T-#k366oaY4ze95A~9Xjr1HNnU6PBv0o_V1@kP)C$ec
zn^jGJ6EFi=p*v+Q_XKXM6^uERd`eZ{0jBmBqG(PH;D(uA`#;f+Y(4&~GH}(fzmr~S
z9ypPag>mxS0%%HL##3dBL<~w*S39!9mkmI}v4LCO>Jw-A+<L`{Zy_IajfW;YgAkGU
zxG?bzFhd!y+AYiK^KUPGNBktBTb~k(p%LoXV5Tws)twUjQZ68w{mN`_30YF}g?ZVH
z(Kqdg6dCYtQiIMP*6)nH>6<@KL5T(?-n!2a%NJ*Yb&6Ib9r3iv9wG^|X~Y%i9Q^Ul
zrdMSV21fckJZ|Cl)~X}{ABaQ00!hth^6PWCGwU!-t`s=9bbjHsrWMf#rkRe1$ZoMH
zeJ?@h+qWD1fA5EsQ+&n3=i|SkF+$VfSVy9qbL&K7BA@3K!MmSm$<p*qaCYk?^x0Q}
zwQayv=A`{N2qI3`Qx)7Vt-^fKOY0B-%Lu6a$&%=sE_<a9Uh>#{@4m4#`3s?hM-dW3
z^u2P&gn_Qq)QjURsw;B?0}sV7eocsHU=y{>Yyxbyd7s{wfDL%hlNL<{pICPT&C`#d
zzbWWj$KBI#kR_EhZA!oR&toxfMg{VMB};$+y!&J{j#ts+aez0b=q5ViAl1llkK^Gv
ziGCBjP~lh7F`--anRx#P3Bq&0;glJt%U;CZ@IsHZXI`I-*3%Vm+=8A%7Q>l{8xzo4
z$knJ1{wzBaj0l+{g5s4g8oy2J7@*0!tOwouwkEz%H$je}m~l?B6g|_7;N{zXiAPR+
zcJC|wa-}7uZAH)PtCV$ihx2a}zF!dHPLUdY@3H**dy>D9P#9*R^GmGD%JY6$rA298
ztx*nWogJW{E>^x^TVC?rfpe6NjfFyf?zj!~{MSv#ipWw=vYre#T%5vO1|w^9jLVO2
zme?mK6Vmaa1Gvb^doWV-Go54vGp0!$n1yIS2Uu73hzDCoQ?F3)@O0l|WT5sB+N*eu
z9lh`-t@0Rc?0KwtDE>NJ@Hf1*Xx1vBo9iR-ydYi3B~7jl*J|S>e4n^sTRWphLBZ}M
z$vOUvI*TJK1ROqgmc)BjSb=*^T7YATVZOi-0H!gKl9Wrvq*i%eRa|Esm~N08-QOU0
zMc%Eu1G^bwSFDxR1=Lv8@Wn>1nW?@!Gd?!*^eJU3cc~T#I3O{_aslwp!?Qo31BqMb
zGUrupmQ{fjv^|Nkupy7<rjk)g+Q%m}DZUH>HQk534V(8kwxsHWi~u3xsC&rv>>HgZ
z<}rcX8Ta1e6IwjRp)gCs27HR{JQOA^BB<zWMD@o)+;(lZ_Q!QQ&`I5q7Ge3pc1w%L
zE^y|A1qk-1bd(lj2Em2!K|+WC;>@FL+?Zg*3Va6v{v}2SN%LIem?cI6?z6C+_Q`0Y
zG5ZDHP;F)0Q8d{neD6MEy!k*jF8498C>KM|h^;)IiFWF5)T+RcI|Ux$R_HxSs~y2B
z0@b{<6v`v+@3^|^UqfKV{@B4tVkS88h6&;K(sY6mknKKL4{n@F5tkv04>6%sNs3SQ
z;{6KdW=aM;Y`<r_q$$w2j_4T6^#25Z$<Nm#2+q(hfJHJEvoZ;XrzSl|xJnWSKyu~o
z#8R8$XV(4F7+&;ev8@O%Fb=MhBy5S@wvka5>}Jxpn^o$LLm<@X`gIU1-eCI^YIjg)
z>g0%M5$*^t)(s}DmihoKs<8&TE52Y{JFI}^w^3!li~I8&8*#;FB!{>joLWPTvJ$j4
zsNlI9foK1sZD&@9R?XRxHy1xVsu|A~R>&k?W!w}bmYo`$GVb2bjw(vs0v6&^b-2)l
z|J?aM5hbkQw-uA4e#}o(&WnoI2R>m4))W>JjhWK3UQA)4<KR87&4W(J89*!|4op$!
zJ6y4+4V!899!@+#xWtQH$|v>~xh7Y-3#FN2PfT|18;Z8SQW>?uf7ys7#h{X!vgjqW
z{zXCen6F;z8NQwb|5DhPFNErf^4e#y3vzk4`$Y%V3E-0V&4?BwoL*j}JHIie#qHFZ
z9*FTm+YxEG!ERG6&$C{#<qr?n;e#gWu}!#-lP(LC%Fbw4yh_W1@bO2)m#Qp`XI=V>
zYkN&`=&xPr1#ES?&FtYOTmeqaKh>q1p*sQn74*}Wv@zp*q7_IOmU0?x>525E2$m<&
zVM1*K8)9v-S5di<4r-HlQGQ`e;K)?(<8$Rv<@*^@+1@GAw$#V(wDHu|F3#i_FWl`C
z%0SnKFsyQ}sjr6G@Q8PXuKtnbXq-D$L3t_X`}7Tt*_LIKsxw&I==Dd8lp<kejLn+S
zxeFZyzSQd;j#;Zr{s8MM+9|TAL>#*pgL~GwTg0_ukq24%f!dSf+K%g@DJciRa{ncp
zn>e|}F|XOQH*`<^uRGPR5|H;$3iVP?#T^!8Zr6X)R8fi8wImTi7Wo4{8Z>7_DMkr6
zLc0ntz$)iR+HGgRpQZo5_dnBnUt4>RwJBbu$%h}gUlfW+!`*S%W6GdI)>R5xD>(l*
z@t0j~*<JmWJSIe%KpN#@(zn7;_FBKivg=)v{$Ti#D?Y`{F^71r_`X<h2MxKi3S!kA
z3W=eALSAo{#qG|gkYqlagUcQKlq^;O^}Dwc+Rn!Lp?XAMr2s#fC|9g8(BS}2p{i4`
zct;8%Gi%-6UnXkusFa=;SHr}m&`RnX@RdOyulmJJ7X(r9UYoi8N|UXq5;b;oJ1EM}
z0zsCvex@Ja2V-c4q2czNs_)eo!m0<ZOnZaMP&C)+3m+DAvTGioM|tyfAos6y%`l0=
zS$n&q_2KbU#9DN{yWq)jTViQTvREQGrB_^^-n}dMqF(#KmA~h{1?z0x=>A}&GIiUv
z6NA19o0x3AchL;nZ(%7FNOkfe64@%$wsJUXta-FG{o5~yYj}XNm-PPW!>rR*5$?-V
z>p6CYEUWI0zVRuIJGkrcQKk^Gd~rXbp&<bW7+YKwyQHy<%}}8i$VaLYQf@caC-buW
zkS0&>?m=%bz$Z+zi??(ooBh`NgII4H>D4!<<+{p!ulJ@nO-AvBc!A~pE-vesTw(dT
zd(g!PJZ@P7iSBj561!dple>q=@mz|2VBql<Gkq+b6k%8n!3E@G7Wc_%vwfi(lVXZa
zjR#)Kx7f~0+Ew*^zWDLlLx$s3ANWRva)FoUz?h7fi52p=g75$y{4i@U$+v*1ZK}Vr
zVpOj?Vu084*d|!?h7Zt;^-^KBYk86DxAMtvc^N1Xc?Xc4CE>oz!sMa?^uC2>AtQQ(
z2Yf`$Z1>(0PemD{ZD{qlFkv|JPb_9r09M@gOdi+pQukaKt-C!FF|*=|IKpMGD%x5z
zqjl9IAiWw|tUj9TPAt{k`XDv0X6>>Oss60J?*uBBcK!4P=yi?d<$f%&_`W*#^8O~Y
zQ$pK1s-L11Zsn_OzK@#^fz$Bb+P!Fkc5c?AbZEb7_Dfn{+YinrIo1iBA#_Vz0dQaP
zlght%#!*3a7mqJ@>n^DMJtuYg>bKT(xf#oaCTraEE)<hBqQdiJq$gdcc=EwfhhF{#
zD1d5gIKXBQi=OH2A~8M8Km8=)_4(H+!Iau6$c!5|zt?bF1J%$MsfgZpI2<bwT@n$#
zZ9=z~J!&KN9rg&f-z5w~iGvT~(XTBFpSENu=FgZ{-tsMhj`tfexrl{o1M<`=d^s!C
zpd-Ep#)+JAjXmYy>H$}v^5PqFbz47O)!rmBaDG?WnLw;nd}0(hj~7y)z8@oJ4G>AF
zM))$jZi3(KJ1E_S+_q_hEVq8*&#xR4@2|Azf0i)Mjr+(|7ywSaau{FG+GFs7VOca(
zoBt}~uwmr{?Lwm!5diH}XpafkJZb>OJRnMY3c2&8`P;rh%Mb2fp6c1aB7gg5o*SG}
z&7HTmj>c}i6mrj&G8`6s2IjR3W<B#T<gHsKEX_1kGILZlbSLPwobTw<x>SSEFH-yd
zI#{3}&AU?<u1dEg)N~$npgu&K1#7|era?GR)V_&%=Gm`ka4ioFcjqHK0x8m9?J*D5
zKhKgHha#EZ?Mm84S3&P6O}%3w4Hw*FYmF*;D>}xRlNz<AeNZrM{s<*W*IMAwyzG6x
zy=CCQN@3r-jyP3e)5-8XJSvzQlc`54<a@HOigpeh5D2(^%|&Pjv%;gA9P5GoDlA&@
z(p)MxU&Jiga|IjH@EHH>q&ahHpkoCrs0P^P=nFj&t88bi^KI4GrlZG^V99f%<VIbg
zUek2h2E35tJ=sqgXpR*fH^B`C9<7`kT0<?7U^cMqt#sk}aw<r{-I;jz(DAYT2RQ84
zzm9?3-O_IFz)LpDL9#?0TcO#s?6z(e<`re`$XZCYIHmTboMVd26Q<6trVHjj0MmPJ
z*YeOduEdBD3JRZ@ME^eXO#h-<Wl5HwY)H`uyXp`a5G$#G{x#B&7_P2u?9?gZ)EMW{
zQ{-Aa1Iug{CVa3?$>C4<A5c0hO(b_74;ng1;?pVGwNHm?JWf^Ch$8<a96U|{_vb8<
z*<iCry!UAibQ~0Rt<Gl#EQnu(?GnxbHlB$aBh3zitY;!EDd9SNe}`+{0yrY*Nxa(i
zF7u(54bkGkGD?S7HPRr)6V|~idpUY0mZNDiM$?H;_prm>&dAz%9{q7Zg6#`{KX2-y
z_HRa*>Z6Chi!R}B?G>PXkSNBU6pw3cNwCaiMp<sqA#i5y*bsyI(w;(c1Km^R!d0;E
z>;1lqx@<@M_2ASC)RnXBy6;##vn6nrU|n(zhV;$0Cf_}&C%jedc>F!S^{;-U4}6S+
zsuT{bBfM~GLrowMH12&I*GUo%g4S!7+a7(d{Ij#E@S`Xk=UnV$$L!pr;ZR5STFm!J
zrFGD0<}36Ka(mNaBX+ho4SwtQ<6}BPqlb`814+i|Q>piv8XI&^!#-|Dg3PF84$&(x
z!s|c>^Tv6kw?HoM7g>hPYV$WZ{Tj5jbgb!~JsdBLbhXYoY8oH!|9*f|Htj0O5CH~l
z-{RPdfekt=7$$b;OLVpX9Zl_>Z0)dx+~{{U75cV;21oS}2H{P3@0L7a=BoTtlt6R+
z4e>x~9MU}NR<07yO4bn}zAzTK!lS>5eulreH2rh2sDSC~9u^6QeGd{(uL_7A5T_iR
zQ}Gqi-II89z?EwNQ{xtEk<o#EmmNfuTz<}n9>ATMWVf@ZLr<vo=yR%fM%<!KHt_qd
zQho-;V#v|U2&t3FdCJa3cM;eGclyIE*!fjpiFAL*h3R31Y;QY}{iSYIiV-U4qdg_P
zjvf!*SQ52=S?x(X991*^v^Lho?gx^GqGW76i*aeBGdt1iQ>%M)KPA0z?1|p2p#8Mj
zuL)Hc42=tb(?Z___nn{Ue3?9&(Y?zxL_g4hN{E-_ClCeBqSdCSDkE_;KYs)!V=8lm
zCeY-aa5*W$HtntH+w{d3KRAo#md+#|U0MO40_WZxezd`VBqpPwIm^-lo5*oR$QnZ&
z^5-{~bP$ccxU#r6X&KZguV3Z9a!Dr*863N~Y;O{KM~tk{<Of;zaJhV?Oc+R?SV?!N
ztM_z0oS6`-<1t(u{ThljH2A4Cq=DqPaDh}TZr+~l%@P^>XhOcZ8t7!a>uDB}h_9hB
z)+ek`&3_Mcyw1zdJf*QUC$9e}3eg!6S_$Stx8ZV*MRmJ0CYG<!Gsnb>P2bt6eH!Bu
zdS<Xe&Vg5a@e#K1_LfY@L1dtduqc<A>=kLrB_`?*tTboBC_=~Seim-H>*9EqdO*nU
z<$(wmvS<VwO+&9dx-i;an+0{9KgAckZlWIZjSHrZQiOm#3Ajt3`Hi7&#zj>T!y0ZI
zb<<Mf8)@rew<qNnZY_KgYwKNyTYOQ8ZVugOWu$g{M$?{pT2O!krA5czfBAQu;zR$7
z-%=8IR4-XIdxHCU%vZ!XYzuH&YYpQW>^$o7SImb_MbMb}%#<BjrQ~=o!;)jc%gt+Y
zub-Bj(WetscV*+X`7FS<AR>CiOgbQ$g<3Vz;^&q~pGsVBrw#0k31)~Nj39NTSM0$q
zu^%J{u{MUYXED<8W{>O<o{s2X(v{f$R_tyyMf=+@3-kH*YAO*!dJt)FMCycNnu_4H
zNx6kT5UCXpl_E@D?*aVJ?uZKPG2BvMLA_^k(!XdeFYVlSF&iP@0*=RS&)4{f-)Vwb
zwvaYSK6s3Pd9mfNUz;<$UPV`GZ+h=4&IEHAw}rMv9q<1gmY00eJOD2<#5<iBttTzm
zNE}~%XkqJ}sp?-Gggzf6=D|5HOZl=CFjO{LtR5iP*iG0dl-SGWZ7^+I6xDb%)^yZC
znWUzjDE${+eEI2Zs`cj99*4XBsaq3b+UDq#9RVF(;khc&OnT9P4sw+HX;GHybq;%C
z5!QNq>SQlY`u_Xdqk?G6x1x-otBRxiyi~$w&`Sd)JO0@W-%bW<Tv|iy%v7AE$U!mg
z#nu&UX;TCJQ$9?B*e{JeMk@5;b$un_jC&JkvJ%)isi(x{Td-Tr4H$Pyc5ZtdPhI)~
zC*wv|`Ip1Ie~~tMf|vw&x7C9;x)a08y=88D7uM`U1^7&V`NP_dJb1qa^x%X%cN~}0
z*&rN#wN-HMPX*i*mKf^TGE!R=eH>rN{xPI?{)SiJV4(aEcRwFl<aTYbdb`qNyP^fQ
z*MUkB#Dt(`a-hv^-{sMblGF2QFa4r6<AR@kS1xrgHFbEw5-_A@aPULID5(ei&P5tV
z`e|PgJEyoXT5IAsu`aW=s>aqIAvR!#E7b6%2BSR^5rB7A@@fR!!(oB4g}-9o<V5=i
zs>l^G$17Ij48`Y2U++__SD19vLoNR7x#AOEze$XV6=gocBG-QlA6ziKc##FchFig2
z31-WnYmC$dK3~V6pS4%3G;Zo3Z?F4V4_^Ne#;199ND2NQZy5alPD?@F1O0!KTRL`d
zW4B&w3-dWt9%3p^#+ims_k3)b=`KVjjiI%LX(2?ZuH&hvLs8zRJs@sV-YEgmqm5I$
zfBRU?*1X}Vh6ur&-xj-QvzzZqH}>-=2FUk6aU`F={*4+z0&N}a{SW^Q0zslHNR;~c
z^S|zF$VsZhZy){FL=KVHEF)~zK8R~asotFE;GA+j>N>Pc)efXgz9AW&#PTWMAOWc;
zPE@%VFPp%p;W4m&c{kAnzeeN{$>(v$y+Ws^asZmWtAiV`WtV|3C$o6Rt^0#ZVx^8W
zamNkbrNq5=H*W#&@)jzb2gLkVwh_<-#NY?@lcuVGeR=G6Be<U~kAFkLk4YEuOnKPw
zxYM_%AhfM#0hl6sHT~AmlNG{mFSW`%l{CMli8E~DfU1g_oie<5C<4c*h+(V?uH^jh
z%OANsN^6nnw=2QmsX@Xf7xG7n5R)y$dZH|L<81KlU+;!h$^iUAmG%P(B-JkD=P!tP
zi$=d>V;a-z0glcaETf5UoC4_<!CwG~EqHvaEyHDMA_p3wa>&b+f~Bz(!p`0A*?+nW
ze=$9YO+9YWzC6>b_?`%A3|DEH&v;MuIKD0K8!s~<+Z{T5M7v-<D!G@43}l%F3axMs
zR|X-pS-;u7$-|P?6Nja2-GbTWv2zb-f>@4h7u(nFJ+-(zoE@|=nP-xdeE41JZvIa8
z;*E5;3AkO(+KOI~weoZjqzg>2ADEb&m7$?}2M$sM`n?0KED5`xnR^1UeOkOacPv%P
zPJQw6H|9EZNJ)==%{7@qn%rE|%-;a!=YAtN8BqU=!XGi1L>z##_~U>^TCy~%kuY<N
z_5O|wvuX&&w+335-*g3|s^BRuaLmm_g|z{c2Wam~s^C6I250n#ZOC_dZjK}P94=mx
zbOJmA7qh&mdvJ{am0Vq|4Le_*Z`oJ`%;yhiyKn)veIr04-_v6;|0D?BFFGLEi+Oi2
zP{tI&oqCkYv1VK7sOOJBBQSEOO|*gb5_#-&^<4Fz{wwW|f8u$0!q*VDw>x^?6q&3g
z${moMfR17nIhp2vxX@TkQ1jFAetOnn3>h3X(r01~{J~fL4veAiK##aU-*XPI9vc-2
zqq$!yy6Uhz7G94yVL-OBNMYgi?YV2g1J<S(HnDPf*RQ1z<Lh2|HUm#V5zaeqVZsjE
zjw@R?Eqh?5hFM;sX>9luMngNj36N><@AI#2K5DZ1XBCT=nLreWh|}L<E!TQ1F+a!?
zapXyqKrY}_=e<dHgzjQ8u;K=#^t}T2q7HCU5ZQ35JAsVQ6vX^!gCDhEp`-lPG!v(A
z3dHg^2A(<^sko8qi>m?+H$-10SnH3FTj9s+T?iFW0fc?eXj?`X(-HaRA5m&#T&`<|
zBJsQRx%qp-gj>vpo_OEEn1N#Vj0}0#`z;TSB?DWM!DC<R#e(;J`1g6imj79y^{_%o
zQSL>a1N|y!sTZqw=uE!VDHJHU_oT72e`yVgrnG0OR>0)4h98K4fw<*G?_;7+UUYvu
zay>CJr0usDaHZa~y=Ms$0MAf{gq6bLkwEZY4j8w7B_r1GNWwg)LeC0_7-f*j>(!%S
zX8=ogAtS9g_o8w0zU75RJvs^}S=Dz92BQnPaw)hK_;%9N`@kx8XWX{02D@y}1_a+b
zHu@O@{`)CnZK1v04gJ51V3NExgP~Z?Gth2-V4_d|5J<^0$r$^ZXXfb{R|7K;;;(p3
z5PdKmFc+at7u8lHgbmqRGdv>m<Bd+Ixxzp&kBnnBRUY(w{fsD-Oph*3Po6ACNw=J}
z4<^zRWam9XVeD{{Jh7XE_sRC2FJ!ti{wS7h<Dd=}S$prZ{#n64*QH~|m9*wXmtU!n
zhG7l1s@^q7?;b$3*ed{Tc%6l?PatzTR+*c;u{`BMxKuT)%%glGrvipq^#OTxeP%dZ
z8T$j(JCYF4xHw~XKFZ~>in;pGA$%?txV|I}C4qMk8R_#NMPn|o^P+QXa4%xKXMs4X
zeEtmB$HQH%I(UvD@ga-v`=;t?!_@tdVJSuQ`!JIz_s@eSM>v)*TS4%?Cfa+2#e*4!
z0NC7`e4fHW?fLJ=fsT4NZR*RjCRgp0Ib0m31?P?VhBB(-m@m5U&iW_z)tfvWbSE^-
z@=&6$@fI&;>WT@S#AIS|h8(oK7qk!Gx57dx`${LF-%nyQm&C&H^Lg9p43kL8;JH9K
z=9{|R?@$=a^0QZ_A3WT#jCoZY4&20wBg4yqIr|nz+qiLQG8<zs>n*l3Tz0}NPWmvN
z`jtJwBX$;1Vn>EdDwm3x{&@_>A<|6VdxZ6^%p1n4*q{EagwtOm^3#V^T@UQpq|=k>
zecvkPKK#w!v9vV@YL|f=wynRHyN&hKsds6^IluXLBXq_q{%Mf!(R5rsB`%??#Pv_=
z7Jfmj2P^p{_rJ-g%kM|zV#d!U{2L*Yy|@6o_1{ggF6#eeC85|kZ+U0JYj>)KqzVEo
ze3VRfD?W+dv0GKN-qk@EYW@<_pT4GD##lSrCrYCtRxk7<Ht+NVAJRFXl7jHd7j`qw
zyFVGtaqs^VI$SwY`%jL3-ONFxuPME!nQw^A(?qIMF_=SM0XT~4tq%Re>3(SU?@!-@
zj;2F#C$t-w%)E2;z}ihe)pSB;7ezB3iWxr`%}x%=6?K|cUXx$>=O@7S5{{#`u6ADj
zC)ZutfxPl>54NR-{JMYX{2j=DnDhUe{tjhfCI1;kQqu;ByuMK3WZ;Q(i$CDW&T_MT
z{VK$hk?xK(aGSJf1i|Bxjn%oiT^qdr+^j7$5{fq`POrkO5KD+pTR!1|(x~jTUB()V
zgK`oH4{+4=@2?(ybO*9K?|D9iE!02_o`sru12Rp8HsW;gek`y8)L<y;JK(DPH0k1J
z=z$p8)Bp#C2k@+U&$HtHkz**@5CalM)xx5!;8wm-3Y%kTu1N-BGAQ58jJ{gc{AORg
zuWEC9l8%&X<`Vcj6v+x9B?W9VS!c$|?mfUc&m%sHNAj+Ydp8=nJa2+c!UsB>;w9MB
zFWmytxsS_}I0SUz7^1no7K4<uOxIIAMkL-0>Zl@Hlbal{r-qyWZ2VwT^vkc>c=b?c
zVAnaZ^Z_yB!cDTC01p6GgaGN!!JZONPr-zOc-TqHfI+L+oU0y4!88HA<d-aF=7(td
zR@EVuk+(W%-q;awg}v{u#6Hi5tPIlBjDAXPQv0*D=<(^k$ZSnGZC=duKdo6GQ04wG
zW>hz2eYNU!epOE}?DoZRrzJ1G&-uO&$z>x(0U=5%jdQ@x)_M~5ssYeg)0;47#u{dd
z^nruGEq&*h#k<GX!K-t5^MCo|Tjp4y9dyJSRc{J@>ig@|ipkszH=aLX7SNh3Hd2!A
zK%_QfLPyt`O!$R(CJsT4I2f`cDR$7O*z^+vOu~}IC`)jVC?Ybb<ayo8A^GH)W*V;r
zdbfrN5M0_GU1=$rg!U!gjjXq4W*u4;_jE6C8oakuli7%<dNSLMq{e|dp2;t&MB2`G
zFtr3TxBN5hp9fO%COBkwjuB#L8qr~RC4Nml+#rqBI~cBl5WsIg7}i%<<r;G?9z|H>
z?P0z@UckG=kd!_!J>i2PD|hiX#6cB!C}r1QY*27#3Uk~j#^txJq5jLW&s%B1w+jBL
zw>$?pw9~!rXA1;@0R6Ff0T!uyHAG5baLP4oyCpo*Z#^TlIfvRJ_l4#M7sGsl-Q|h6
z)ALA3<OCVcG7dlBOeKTNEk`}&r0Cu=g0zS5+&WWh+{}*d_pa*nUz+$b7W1pn$w0V2
zoqnkqoGU*GH@12eDU(H4g+&O}h@YhbG^Cq|Uq8kUud)4E9jIh4D70$se8e&A(U>HS
z!U2(!54?~V7bJ;_ih)WaeT6WDd+ZbDBLV{WmtNADZ~to{4IZNcMd4(TV~An=j?60g
z1y9#ZtqKV&hy@2!_9V*0ixmmDv_C#v9xjI5AI1FF?9I6BdI~3HvH%b44cjX%8<=_Y
zyjkTli4o@0n)S*K1kgq)*2#89m0Y)vzD#T<c?CY>!Qbf6AF$mJNHeuOwf2^W`oG*k
z&f;1t5$Ti|pWGyQbfT;P1L~S5{un<&5vCtN=+0F;J%RDA3#oJ<>V~0^sS?37R^O45
zPS1ncutf#dFCMT2KatH$&qpCwYwjGgtbSm|n^PF5M?nIMx#o$*riq#yDS-qvqsCCU
z7Bk9|IF@)i$>;yFN(0_<&*`qP9N1sm+xPLh1>?6Gr<UWrsBeQj&A8l)lILEW*2R8c
z$pRhgqvE+%NNK7Z`0`1XN*X^tmacTU|42l^xLPZs;Z+_RhCtr?8!|s^AOXM9!B2ZK
z)1!LZ?=>3&I8skszYx1hP!BHWI&tkmz@wN@=MGo7Ise>X;(5H~90b=dTDFw^_vImM
zBp>XzYE_MW7IW!$oemD~gFa;}F6~*wj}H$OLDC`(#>30YJlgaEqMj8q3X4nc6D+!6
zOKY+9^;aJ#*6N=lE+!yy%XjuCr?_u+%p0>gYgsGKee4I*mFpz#i7T4`FY!)-#z}^q
zFMs7%KeL%8D**emo~4@g{J)-->b3omU4buHc(2(>Rnb?rej!OuMa=m+tC9j7M@iae
zDau!IF>JDUjQ|m6qsm(e+n>agA)<^_P<9o78rqdRv*is!1oQj<c~j+m{|+UguJThK
z@BLSGUJFD@9~3f|YqI~@OY<}*xcb{#Yp93sb=FZDVnufKYi}(f_`xTh7xHc0`^Vw-
zJU_y*3EJ!RT`kMIR*?Ew`$_0op&`HBQx&WYvn~ElMVsdT7s9vjBV7N->#0yH`Z?=<
z#*PEh`{x3J87Hv-w-UiIC#$V2PM{ik_7=ARU2FKSif?VL%SH~Gg@GN&ilCE^NH?>r
z@O#VPT&(IzN&hQJrUmI5pimvnK7&9TFu?4+1v?s;V|+c<7e)`i3UB(Em#Ii4y8V<r
zevk%{T@f*=5vs4rhy8km?4R&1YnBsU8vz(HGW+CnHXMz;+lh2&zGh9pm~e~fy}#>%
zs%ZS5mr;!}O*-#m|MeS4-b8meIB+xm+lEk{!dT^h8?OKT{0AfX;49<Xh?#o38CEsB
Q2mUkCGu3^kZ6EP}0JE0RNB{r;

literal 0
HcmV?d00001

diff --git a/pom.xml b/pom.xml
index 7ba22d2a076..08fd00ff6ec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -891,6 +891,14 @@
                                         <skip>true</skip>
                                     </run>
                                 </image>
+
+                                <image>
+                                    <name>gdcc/dev_nginx:unstable</name>
+                                    <build>
+                                        <contextDir>${project.basedir}/modules/nginx</contextDir>
+                                    </build>
+                                </image>
+
                             </images>
                             <autoCreateCustomNetworks>true</autoCreateCustomNetworks>
                         </configuration>

From 18cd587a46ad7770109d501520126bd185517b7c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 2 Nov 2023 12:06:08 -0400
Subject: [PATCH 660/922] A few more improvements, borrowed from @qqmyers QDR
 fork: eliminates reindexing the parent dataverse when adding guestbooks,
 stops the update dataverse command from unnecessarily reindexing the
 underlying datasets in some cases, makes that reindex truly async. in the
 onSuccess part of the command. (#9635)

---
 .../command/impl/UpdateDataverseCommand.java  |  31 +-
 .../dataverse/ingest/IngestServiceBean.java   | 463 +++++++++---------
 .../iq/dataverse/search/IndexServiceBean.java |   1 +
 3 files changed, 259 insertions(+), 236 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index 56c76f04c05..fe9415f39f9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -32,6 +32,8 @@ public class UpdateDataverseCommand extends AbstractCommand<Dataverse> {
 	private final List<DatasetFieldType> facetList;
         private final List<Dataverse> featuredDataverseList;
         private final List<DataverseFieldTypeInputLevel> inputLevelList;
+        
+        private boolean datasetsReindexRequired = false; 
 
 	public UpdateDataverseCommand(Dataverse editedDv, List<DatasetFieldType> facetList, List<Dataverse> featuredDataverseList, 
                     DataverseRequest aRequest,  List<DataverseFieldTypeInputLevel> inputLevelList ) {
@@ -74,9 +76,13 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
                 }
             }
             
-            DataverseType oldDvType = ctxt.dataverses().find(editedDv.getId()).getDataverseType();
-            String oldDvAlias = ctxt.dataverses().find(editedDv.getId()).getAlias();
-            String oldDvName = ctxt.dataverses().find(editedDv.getId()).getName();
+            Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
+            
+            DataverseType oldDvType = oldDv.getDataverseType();
+            String oldDvAlias = oldDv.getAlias();
+            String oldDvName = oldDv.getName();
+            oldDv = null; 
+            
             Dataverse result = ctxt.dataverses().save(editedDv);
             
             if ( facetList != null ) {
@@ -101,6 +107,14 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
                 }
             }
             
+            // We don't want to reindex the children datasets unnecessarily: 
+            // When these values are changed we need to reindex all children datasets
+            // This check is not recursive as all the values just report the immediate parent
+            if (!oldDvType.equals(editedDv.getDataverseType())
+                || !oldDvName.equals(editedDv.getName())
+                || !oldDvAlias.equals(editedDv.getAlias())) {
+                datasetsReindexRequired = true;
+            }
             
             return result;
 	}
@@ -110,9 +124,16 @@ public boolean onSuccess(CommandContext ctxt, Object r) {
         
         // first kick of async index of datasets
         // TODO: is this actually needed? Is there a better way to handle
+        // It appears that we at some point lost some extra logic here, where
+        // we only reindex the underlying datasets if one or more of the specific set
+        // of fields have been changed (since these values are included in the 
+        // indexed solr documents for dataasets). So I'm putting that back. -L.A.
         Dataverse result = (Dataverse) r;
-        List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
-        ctxt.index().asyncIndexDatasetList(datasets, true);
+        
+        if (datasetsReindexRequired) {
+            List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
+            ctxt.index().asyncIndexDatasetList(datasets, true);
+        }
         
         return ctxt.dataverses().index((Dataverse) r);
     }  
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 40dc3d6fdd6..ac5c8e44a72 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -168,55 +168,55 @@ public class IngestServiceBean {
     // It must be called before we attempt to permanently save the files in 
     // the database by calling the Save command on the dataset and/or version.
     public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
-                                                   List<DataFile> newFiles,
-                                                   DataFile fileToReplace,
-                                                   boolean tabIngest) {
-		List<DataFile> ret = new ArrayList<>();
-
-		if (newFiles != null && newFiles.size() > 0) {
-			// ret = new ArrayList<>();
-			// final check for duplicate file names;
-			// we tried to make the file names unique on upload, but then
-			// the user may have edited them on the "add files" page, and
-			// renamed FOOBAR-1.txt back to FOOBAR.txt...
+            List<DataFile> newFiles,
+            DataFile fileToReplace,
+            boolean tabIngest) {
+        List<DataFile> ret = new ArrayList<>();
+
+        if (newFiles != null && newFiles.size() > 0) {
+            // ret = new ArrayList<>();
+            // final check for duplicate file names;
+            // we tried to make the file names unique on upload, but then
+            // the user may have edited them on the "add files" page, and
+            // renamed FOOBAR-1.txt back to FOOBAR.txt...
             IngestUtil.checkForDuplicateFileNamesFinal(version, newFiles, fileToReplace);
-			Dataset dataset = version.getDataset();
-
-			for (DataFile dataFile : newFiles) {
-				boolean unattached = false;
-				boolean savedSuccess = false;
-				if (dataFile.getOwner() == null) {
-					unattached = true;
-					dataFile.setOwner(dataset);
-				}
+            Dataset dataset = version.getDataset();
+
+            for (DataFile dataFile : newFiles) {
+                boolean unattached = false;
+                boolean savedSuccess = false;
+                if (dataFile.getOwner() == null) {
+                    unattached = true;
+                    dataFile.setOwner(dataset);
+                }
 
-				String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier());
-				String driverType = DataAccess.getDriverType(storageInfo[0]);
-				String storageLocation = storageInfo[1];
-				String tempFileLocation = null;
-				Path tempLocationPath = null;
-				if (driverType.equals("tmp")) {  //"tmp" is the default if no prefix or the "tmp://" driver
-					tempFileLocation = FileUtil.getFilesTempDirectory() + "/" + storageLocation;
+                String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier());
+                String driverType = DataAccess.getDriverType(storageInfo[0]);
+                String storageLocation = storageInfo[1];
+                String tempFileLocation = null;
+                Path tempLocationPath = null;
+                if (driverType.equals("tmp")) {  //"tmp" is the default if no prefix or the "tmp://" driver
+                    tempFileLocation = FileUtil.getFilesTempDirectory() + "/" + storageLocation;
 
-					// Try to save the file in its permanent location:
-					tempLocationPath = Paths.get(tempFileLocation);
-					WritableByteChannel writeChannel = null;
-					FileChannel readChannel = null;
+                    // Try to save the file in its permanent location:
+                    tempLocationPath = Paths.get(tempFileLocation);
+                    WritableByteChannel writeChannel = null;
+                    FileChannel readChannel = null;
 
-					StorageIO<DataFile> dataAccess = null;
+                    StorageIO<DataFile> dataAccess = null;
 
-					try {
-						logger.fine("Attempting to create a new storageIO object for " + storageLocation);
-						dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
+                    try {
+                        logger.fine("Attempting to create a new storageIO object for " + storageLocation);
+                        dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
 
-						logger.fine("Successfully created a new storageIO object.");
-						/*
+                        logger.fine("Successfully created a new storageIO object.");
+                        /*
 						 * This commented-out code demonstrates how to copy bytes from a local
 						 * InputStream (or a readChannel) into the writable byte channel of a Dataverse
 						 * DataAccessIO object:
-						 */
+                         */
 
-						/*
+ /*
 						 * storageIO.open(DataAccessOption.WRITE_ACCESS);
 						 * 
 						 * writeChannel = storageIO.getWriteChannel(); readChannel = new
@@ -225,9 +225,9 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 						 * long bytesPerIteration = 16 * 1024; // 16K bytes long start = 0; while (
 						 * start < readChannel.size() ) { readChannel.transferTo(start,
 						 * bytesPerIteration, writeChannel); start += bytesPerIteration; }
-						 */
+                         */
 
-						/*
+ /*
 						 * But it's easier to use this convenience method from the DataAccessIO:
 						 * 
 						 * (if the underlying storage method for this file is local filesystem, the
@@ -235,214 +235,215 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 						 * 
 						 * Files.copy(tempLocationPath, storageIO.getFileSystemLocation(),
 						 * StandardCopyOption.REPLACE_EXISTING);
-						 */
-						dataAccess.savePath(tempLocationPath);
-
-						// Set filesize in bytes
-						//
-						dataFile.setFilesize(dataAccess.getSize());
-						savedSuccess = true;
-						logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
-
-                                            // TODO: reformat this file to remove the many tabs added in cc08330
-                                            extractMetadataNcml(dataFile, tempLocationPath);
-
-					} catch (IOException ioex) {
-                    logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")");
-					} finally {
-						if (readChannel != null) {
-							try {
-								readChannel.close();
-							} catch (IOException e) {
-							}
-						}
-						if (writeChannel != null) {
-							try {
-								writeChannel.close();
-							} catch (IOException e) {
-							}
-						}
-					}
+                         */
+                        dataAccess.savePath(tempLocationPath);
+
+                        // Set filesize in bytes
+                        //
+                        dataFile.setFilesize(dataAccess.getSize());
+                        savedSuccess = true;
+                        logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
+
+                        // TODO: reformat this file to remove the many tabs added in cc08330
+                        extractMetadataNcml(dataFile, tempLocationPath);
+
+                    } catch (IOException ioex) {
+                        logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")");
+                    } finally {
+                        if (readChannel != null) {
+                            try {
+                                readChannel.close();
+                            } catch (IOException e) {
+                            }
+                        }
+                        if (writeChannel != null) {
+                            try {
+                                writeChannel.close();
+                            } catch (IOException e) {
+                            }
+                        }
+                    }
 
                     // Since we may have already spent some CPU cycles scaling down image thumbnails, 
-					// we may as well save them, by moving these generated images to the permanent
-					// dataset directory. We should also remember to delete any such files in the
-					// temp directory:
-					List<Path> generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()),
-							storageLocation);
-					if (generatedTempFiles != null) {
-						for (Path generated : generatedTempFiles) {
-							if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to
-												// save the main file!
-								logger.fine("(Will also try to permanently save generated thumbnail file "
-										+ generated.toString() + ")");
-								try {
-									// Files.copy(generated, Paths.get(dataset.getFileSystemDirectory().toString(),
-									// generated.getFileName().toString()));
-									int i = generated.toString().lastIndexOf("thumb");
-									if (i > 1) {
-										String extensionTag = generated.toString().substring(i);
-										dataAccess.savePathAsAux(generated, extensionTag);
-										logger.fine(
-												"Saved generated thumbnail as aux object. \"preview available\" status: "
-														+ dataFile.isPreviewImageAvailable());
-									} else {
-										logger.warning(
-												"Generated thumbnail file name does not match the expected pattern: "
-														+ generated.toString());
-									}
-
-								} catch (IOException ioex) {
-									logger.warning("Failed to save generated file " + generated.toString());
-								}
-							}
-
-							// ... but we definitely want to delete it:
-							try {
-								Files.delete(generated);
-							} catch (IOException ioex) {
-								logger.warning("Failed to delete generated file " + generated.toString());
-							}
-						}
-					}
-					// Any necessary post-processing:
-					// performPostProcessingTasks(dataFile);
-				} else {
-					try {
-						StorageIO<DvObject> dataAccess = DataAccess.getStorageIO(dataFile);
-						//Populate metadata
-						dataAccess.open(DataAccessOption.READ_ACCESS);
-						//set file size
-						logger.fine("Setting file size: " + dataAccess.getSize());
-						dataFile.setFilesize(dataAccess.getSize());
-						if(dataAccess instanceof S3AccessIO) {
-							  ((S3AccessIO<DvObject>)dataAccess).removeTempTag();
-						}
-					} catch (IOException ioex) {
-						logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " ("
-								+ ioex.getMessage() + ")");
-					}
-					savedSuccess = true;
-				}
+                    // we may as well save them, by moving these generated images to the permanent
+                    // dataset directory. We should also remember to delete any such files in the
+                    // temp directory:
+                    List<Path> generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()),
+                            storageLocation);
+                    if (generatedTempFiles != null) {
+                        for (Path generated : generatedTempFiles) {
+                            if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to
+                                // save the main file!
+                                logger.fine("(Will also try to permanently save generated thumbnail file "
+                                        + generated.toString() + ")");
+                                try {
+                                    // Files.copy(generated, Paths.get(dataset.getFileSystemDirectory().toString(),
+                                    // generated.getFileName().toString()));
+                                    int i = generated.toString().lastIndexOf("thumb");
+                                    if (i > 1) {
+                                        String extensionTag = generated.toString().substring(i);
+                                        dataAccess.savePathAsAux(generated, extensionTag);
+                                        logger.fine(
+                                                "Saved generated thumbnail as aux object. \"preview available\" status: "
+                                                + dataFile.isPreviewImageAvailable());
+                                    } else {
+                                        logger.warning(
+                                                "Generated thumbnail file name does not match the expected pattern: "
+                                                + generated.toString());
+                                    }
+
+                                } catch (IOException ioex) {
+                                    logger.warning("Failed to save generated file " + generated.toString());
+                                }
+                            }
+
+                            // ... but we definitely want to delete it:
+                            try {
+                                Files.delete(generated);
+                            } catch (IOException ioex) {
+                                logger.warning("Failed to delete generated file " + generated.toString());
+                            }
+                        }
+                    }
+                    // Any necessary post-processing:
+                    // performPostProcessingTasks(dataFile);
+                } else {
+                    // This is a direct upload 
+                    try {
+                        StorageIO<DvObject> dataAccess = DataAccess.getStorageIO(dataFile);
+                        //Populate metadata
+                        dataAccess.open(DataAccessOption.READ_ACCESS);
+                        //set file size
+                        logger.fine("Setting file size: " + dataAccess.getSize());
+                        dataFile.setFilesize(dataAccess.getSize());
+                        if (dataAccess instanceof S3AccessIO) {
+                            ((S3AccessIO<DvObject>) dataAccess).removeTempTag();
+                        }
+                    } catch (IOException ioex) {
+                        logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " ("
+                                + ioex.getMessage() + ")");
+                    }
+                    savedSuccess = true;
+                }
+
+                logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
+                boolean belowLimit = false;
+
+                try {
+                    //getting StorageIO may require knowing the owner (so this must come before owner is potentially set back to null
+                    belowLimit = dataFile.getStorageIO().isBelowIngestSizeLimit();
+                } catch (IOException e) {
+                    logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
+                }
 
-				logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
-				boolean belowLimit = false;
-
-				try {
-					//getting StorageIO may require knowing the owner (so this must come before owner is potentially set back to null
-					belowLimit = dataFile.getStorageIO().isBelowIngestSizeLimit();
-				} catch (IOException e) {
-					logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
-				} 
-
-				if (savedSuccess && belowLimit) {
-					// These are all brand new files, so they should all have
-					// one filemetadata total. -- L.A.
-					FileMetadata fileMetadata = dataFile.getFileMetadatas().get(0);
-					String fileName = fileMetadata.getLabel();
-
-					boolean metadataExtracted = false;
-                                        boolean metadataExtractedFromNetcdf = false;
-					if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
-						/*
+                if (savedSuccess && belowLimit) {
+                    // These are all brand new files, so they should all have
+                    // one filemetadata total. -- L.A.
+                    FileMetadata fileMetadata = dataFile.getFileMetadatas().get(0);
+                    String fileName = fileMetadata.getLabel();
+
+                    boolean metadataExtracted = false;
+                    boolean metadataExtractedFromNetcdf = false;
+                    if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
+                        /*
 						 * Note that we don't try to ingest the file right away - instead we mark it as
 						 * "scheduled for ingest", then at the end of the save process it will be queued
 						 * for async. ingest in the background. In the meantime, the file will be
 						 * ingested as a regular, non-tabular file, and appear as such to the user,
 						 * until the ingest job is finished with the Ingest Service.
-						 */
-						dataFile.SetIngestScheduled();
-					} else if (fileMetadataExtractable(dataFile)) {
-
-						try {
-							// FITS is the only type supported for metadata
-							// extraction, as of now. -- L.A. 4.0
-                                                        // Note that extractMetadataNcml() is used for NetCDF/HDF5.
-							dataFile.setContentType("application/fits");
-							metadataExtracted = extractMetadata(tempFileLocation, dataFile, version);
-						} catch (IOException mex) {
-							logger.severe("Caught exception trying to extract indexable metadata from file "
-									+ fileName + ",  " + mex.getMessage());
-						}
-						if (metadataExtracted) {
-							logger.fine("Successfully extracted indexable metadata from file " + fileName);
-						} else {
-							logger.fine("Failed to extract indexable metadata from file " + fileName);
-						}
-                                        } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) {
-                                            try {
-                                                logger.fine("trying to extract metadata from netcdf");
-                                                metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version);
-                                            } catch (IOException ex) {
-                                                logger.fine("could not extract metadata from netcdf: " + ex);
-                                            }
-                                            if (metadataExtractedFromNetcdf) {
-                                                logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName);
-                                            } else {
-                                                logger.fine("Failed to extract indexable metadata from netcdf file " + fileName);
-                                            }
+                         */
+                        dataFile.SetIngestScheduled();
+                    } else if (fileMetadataExtractable(dataFile)) {
 
-                                        } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
+                        try {
+                            // FITS is the only type supported for metadata
+                            // extraction, as of now. -- L.A. 4.0
+                            // Note that extractMetadataNcml() is used for NetCDF/HDF5.
+                            dataFile.setContentType("application/fits");
+                            metadataExtracted = extractMetadata(tempFileLocation, dataFile, version);
+                        } catch (IOException mex) {
+                            logger.severe("Caught exception trying to extract indexable metadata from file "
+                                    + fileName + ",  " + mex.getMessage());
+                        }
+                        if (metadataExtracted) {
+                            logger.fine("Successfully extracted indexable metadata from file " + fileName);
+                        } else {
+                            logger.fine("Failed to extract indexable metadata from file " + fileName);
+                        }
+                    } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) {
+                        try {
+                            logger.fine("trying to extract metadata from netcdf");
+                            metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version);
+                        } catch (IOException ex) {
+                            logger.fine("could not extract metadata from netcdf: " + ex);
+                        }
+                        if (metadataExtractedFromNetcdf) {
+                            logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName);
+                        } else {
+                            logger.fine("Failed to extract indexable metadata from netcdf file " + fileName);
+                        }
+
+                    } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
                         // Make sure no *uningested* tab-delimited files are saved with the type "text/tab-separated-values"!
                         // "text/tsv" should be used instead: 
                         dataFile.setContentType(FileUtil.MIME_TYPE_TSV);
                     }
-				}
+                }
                 if (unattached) {
                     dataFile.setOwner(null);
                 }
-				// ... and let's delete the main temp file if it exists:
-				if(tempLocationPath!=null) {
-    				try {
-	    				logger.fine("Will attempt to delete the temp file " + tempLocationPath.toString());
-			    		Files.delete(tempLocationPath);
-				    } catch (IOException ex) {
-					    // (non-fatal - it's just a temp file.)
-    					logger.warning("Failed to delete temp file " + tempLocationPath.toString());
-	    			}				
-				}
-				if (savedSuccess) {
-					// temp dbug line
-					// System.out.println("ADDING FILE: " + fileName + "; for dataset: " +
-					// dataset.getGlobalId());
-					// Make sure the file is attached to the dataset and to the version, if this
-					// hasn't been done yet:
-					if (dataFile.getOwner() == null) {
-						dataFile.setOwner(dataset);
-
-						version.getFileMetadatas().add(dataFile.getFileMetadata());
-						dataFile.getFileMetadata().setDatasetVersion(version);
-						dataset.getFiles().add(dataFile);
-
-						if (dataFile.getFileMetadata().getCategories() != null) {
-							ListIterator<DataFileCategory> dfcIt = dataFile.getFileMetadata().getCategories()
-									.listIterator();
-
-							while (dfcIt.hasNext()) {
-								DataFileCategory dataFileCategory = dfcIt.next();
-
-								if (dataFileCategory.getDataset() == null) {
-									DataFileCategory newCategory = dataset
-											.getCategoryByName(dataFileCategory.getName());
-									if (newCategory != null) {
-										newCategory.addFileMetadata(dataFile.getFileMetadata());
-										// dataFileCategory = newCategory;
-										dfcIt.set(newCategory);
-									} else {
-										dfcIt.remove();
-									}
-								}
-							}
-						}
-					}
-				}
+                // ... and let's delete the main temp file if it exists:
+                if (tempLocationPath != null) {
+                    try {
+                        logger.fine("Will attempt to delete the temp file " + tempLocationPath.toString());
+                        Files.delete(tempLocationPath);
+                    } catch (IOException ex) {
+                        // (non-fatal - it's just a temp file.)
+                        logger.warning("Failed to delete temp file " + tempLocationPath.toString());
+                    }
+                }
+                if (savedSuccess) {
+                    // temp dbug line
+                    // System.out.println("ADDING FILE: " + fileName + "; for dataset: " +
+                    // dataset.getGlobalId());
+                    // Make sure the file is attached to the dataset and to the version, if this
+                    // hasn't been done yet:
+                    if (dataFile.getOwner() == null) {
+                        dataFile.setOwner(dataset);
+
+                        version.getFileMetadatas().add(dataFile.getFileMetadata());
+                        dataFile.getFileMetadata().setDatasetVersion(version);
+                        dataset.getFiles().add(dataFile);
+
+                        if (dataFile.getFileMetadata().getCategories() != null) {
+                            ListIterator<DataFileCategory> dfcIt = dataFile.getFileMetadata().getCategories()
+                                    .listIterator();
+
+                            while (dfcIt.hasNext()) {
+                                DataFileCategory dataFileCategory = dfcIt.next();
+
+                                if (dataFileCategory.getDataset() == null) {
+                                    DataFileCategory newCategory = dataset
+                                            .getCategoryByName(dataFileCategory.getName());
+                                    if (newCategory != null) {
+                                        newCategory.addFileMetadata(dataFile.getFileMetadata());
+                                        // dataFileCategory = newCategory;
+                                        dfcIt.set(newCategory);
+                                    } else {
+                                        dfcIt.remove();
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
 
-				ret.add(dataFile);
-			}
-		}
+                ret.add(dataFile);
+            }
+        }
 
-		return ret;
-	}
+        return ret;
+    }
     
     public List<Path> listGeneratedTempFiles(Path tempDirectory, String baseName) {
         List<Path> generatedFiles = new ArrayList<>();
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index d6d0be7a17b..4e27465f83f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -420,6 +420,7 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
         }
     }
 
+    @Asynchronous
     public void asyncIndexDatasetList(List<Dataset> datasets, boolean doNormalSolrDocCleanUp) {
         for(Dataset dataset : datasets) {
             asyncIndexDataset(dataset, true);

From 5b1c80047911486813ac9872646fe1c390ff327b Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 2 Nov 2023 12:38:10 -0400
Subject: [PATCH 661/922] Revert "A few more improvements, borrowed from
 @qqmyers QDR fork: eliminates reindexing the" (some things got checked in by
 mistake) This reverts commit 18cd587a46ad7770109d501520126bd185517b7c.

---
 .../command/impl/UpdateDataverseCommand.java  |  31 +-
 .../dataverse/ingest/IngestServiceBean.java   | 463 +++++++++---------
 .../iq/dataverse/search/IndexServiceBean.java |   1 -
 3 files changed, 236 insertions(+), 259 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index fe9415f39f9..56c76f04c05 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -32,8 +32,6 @@ public class UpdateDataverseCommand extends AbstractCommand<Dataverse> {
 	private final List<DatasetFieldType> facetList;
         private final List<Dataverse> featuredDataverseList;
         private final List<DataverseFieldTypeInputLevel> inputLevelList;
-        
-        private boolean datasetsReindexRequired = false; 
 
 	public UpdateDataverseCommand(Dataverse editedDv, List<DatasetFieldType> facetList, List<Dataverse> featuredDataverseList, 
                     DataverseRequest aRequest,  List<DataverseFieldTypeInputLevel> inputLevelList ) {
@@ -76,13 +74,9 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
                 }
             }
             
-            Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
-            
-            DataverseType oldDvType = oldDv.getDataverseType();
-            String oldDvAlias = oldDv.getAlias();
-            String oldDvName = oldDv.getName();
-            oldDv = null; 
-            
+            DataverseType oldDvType = ctxt.dataverses().find(editedDv.getId()).getDataverseType();
+            String oldDvAlias = ctxt.dataverses().find(editedDv.getId()).getAlias();
+            String oldDvName = ctxt.dataverses().find(editedDv.getId()).getName();
             Dataverse result = ctxt.dataverses().save(editedDv);
             
             if ( facetList != null ) {
@@ -107,14 +101,6 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
                 }
             }
             
-            // We don't want to reindex the children datasets unnecessarily: 
-            // When these values are changed we need to reindex all children datasets
-            // This check is not recursive as all the values just report the immediate parent
-            if (!oldDvType.equals(editedDv.getDataverseType())
-                || !oldDvName.equals(editedDv.getName())
-                || !oldDvAlias.equals(editedDv.getAlias())) {
-                datasetsReindexRequired = true;
-            }
             
             return result;
 	}
@@ -124,16 +110,9 @@ public boolean onSuccess(CommandContext ctxt, Object r) {
         
         // first kick of async index of datasets
         // TODO: is this actually needed? Is there a better way to handle
-        // It appears that we at some point lost some extra logic here, where
-        // we only reindex the underlying datasets if one or more of the specific set
-        // of fields have been changed (since these values are included in the 
-        // indexed solr documents for dataasets). So I'm putting that back. -L.A.
         Dataverse result = (Dataverse) r;
-        
-        if (datasetsReindexRequired) {
-            List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
-            ctxt.index().asyncIndexDatasetList(datasets, true);
-        }
+        List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
+        ctxt.index().asyncIndexDatasetList(datasets, true);
         
         return ctxt.dataverses().index((Dataverse) r);
     }  
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index ac5c8e44a72..40dc3d6fdd6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -168,55 +168,55 @@ public class IngestServiceBean {
     // It must be called before we attempt to permanently save the files in 
     // the database by calling the Save command on the dataset and/or version.
     public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
-            List<DataFile> newFiles,
-            DataFile fileToReplace,
-            boolean tabIngest) {
-        List<DataFile> ret = new ArrayList<>();
-
-        if (newFiles != null && newFiles.size() > 0) {
-            // ret = new ArrayList<>();
-            // final check for duplicate file names;
-            // we tried to make the file names unique on upload, but then
-            // the user may have edited them on the "add files" page, and
-            // renamed FOOBAR-1.txt back to FOOBAR.txt...
+                                                   List<DataFile> newFiles,
+                                                   DataFile fileToReplace,
+                                                   boolean tabIngest) {
+		List<DataFile> ret = new ArrayList<>();
+
+		if (newFiles != null && newFiles.size() > 0) {
+			// ret = new ArrayList<>();
+			// final check for duplicate file names;
+			// we tried to make the file names unique on upload, but then
+			// the user may have edited them on the "add files" page, and
+			// renamed FOOBAR-1.txt back to FOOBAR.txt...
             IngestUtil.checkForDuplicateFileNamesFinal(version, newFiles, fileToReplace);
-            Dataset dataset = version.getDataset();
-
-            for (DataFile dataFile : newFiles) {
-                boolean unattached = false;
-                boolean savedSuccess = false;
-                if (dataFile.getOwner() == null) {
-                    unattached = true;
-                    dataFile.setOwner(dataset);
-                }
+			Dataset dataset = version.getDataset();
+
+			for (DataFile dataFile : newFiles) {
+				boolean unattached = false;
+				boolean savedSuccess = false;
+				if (dataFile.getOwner() == null) {
+					unattached = true;
+					dataFile.setOwner(dataset);
+				}
 
-                String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier());
-                String driverType = DataAccess.getDriverType(storageInfo[0]);
-                String storageLocation = storageInfo[1];
-                String tempFileLocation = null;
-                Path tempLocationPath = null;
-                if (driverType.equals("tmp")) {  //"tmp" is the default if no prefix or the "tmp://" driver
-                    tempFileLocation = FileUtil.getFilesTempDirectory() + "/" + storageLocation;
+				String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier());
+				String driverType = DataAccess.getDriverType(storageInfo[0]);
+				String storageLocation = storageInfo[1];
+				String tempFileLocation = null;
+				Path tempLocationPath = null;
+				if (driverType.equals("tmp")) {  //"tmp" is the default if no prefix or the "tmp://" driver
+					tempFileLocation = FileUtil.getFilesTempDirectory() + "/" + storageLocation;
 
-                    // Try to save the file in its permanent location:
-                    tempLocationPath = Paths.get(tempFileLocation);
-                    WritableByteChannel writeChannel = null;
-                    FileChannel readChannel = null;
+					// Try to save the file in its permanent location:
+					tempLocationPath = Paths.get(tempFileLocation);
+					WritableByteChannel writeChannel = null;
+					FileChannel readChannel = null;
 
-                    StorageIO<DataFile> dataAccess = null;
+					StorageIO<DataFile> dataAccess = null;
 
-                    try {
-                        logger.fine("Attempting to create a new storageIO object for " + storageLocation);
-                        dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
+					try {
+						logger.fine("Attempting to create a new storageIO object for " + storageLocation);
+						dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
 
-                        logger.fine("Successfully created a new storageIO object.");
-                        /*
+						logger.fine("Successfully created a new storageIO object.");
+						/*
 						 * This commented-out code demonstrates how to copy bytes from a local
 						 * InputStream (or a readChannel) into the writable byte channel of a Dataverse
 						 * DataAccessIO object:
-                         */
+						 */
 
- /*
+						/*
 						 * storageIO.open(DataAccessOption.WRITE_ACCESS);
 						 * 
 						 * writeChannel = storageIO.getWriteChannel(); readChannel = new
@@ -225,9 +225,9 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 						 * long bytesPerIteration = 16 * 1024; // 16K bytes long start = 0; while (
 						 * start < readChannel.size() ) { readChannel.transferTo(start,
 						 * bytesPerIteration, writeChannel); start += bytesPerIteration; }
-                         */
+						 */
 
- /*
+						/*
 						 * But it's easier to use this convenience method from the DataAccessIO:
 						 * 
 						 * (if the underlying storage method for this file is local filesystem, the
@@ -235,215 +235,214 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 						 * 
 						 * Files.copy(tempLocationPath, storageIO.getFileSystemLocation(),
 						 * StandardCopyOption.REPLACE_EXISTING);
-                         */
-                        dataAccess.savePath(tempLocationPath);
-
-                        // Set filesize in bytes
-                        //
-                        dataFile.setFilesize(dataAccess.getSize());
-                        savedSuccess = true;
-                        logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
-
-                        // TODO: reformat this file to remove the many tabs added in cc08330
-                        extractMetadataNcml(dataFile, tempLocationPath);
-
-                    } catch (IOException ioex) {
-                        logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")");
-                    } finally {
-                        if (readChannel != null) {
-                            try {
-                                readChannel.close();
-                            } catch (IOException e) {
-                            }
-                        }
-                        if (writeChannel != null) {
-                            try {
-                                writeChannel.close();
-                            } catch (IOException e) {
-                            }
-                        }
-                    }
+						 */
+						dataAccess.savePath(tempLocationPath);
+
+						// Set filesize in bytes
+						//
+						dataFile.setFilesize(dataAccess.getSize());
+						savedSuccess = true;
+						logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
+
+                                            // TODO: reformat this file to remove the many tabs added in cc08330
+                                            extractMetadataNcml(dataFile, tempLocationPath);
+
+					} catch (IOException ioex) {
+                    logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")");
+					} finally {
+						if (readChannel != null) {
+							try {
+								readChannel.close();
+							} catch (IOException e) {
+							}
+						}
+						if (writeChannel != null) {
+							try {
+								writeChannel.close();
+							} catch (IOException e) {
+							}
+						}
+					}
 
                     // Since we may have already spent some CPU cycles scaling down image thumbnails, 
-                    // we may as well save them, by moving these generated images to the permanent
-                    // dataset directory. We should also remember to delete any such files in the
-                    // temp directory:
-                    List<Path> generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()),
-                            storageLocation);
-                    if (generatedTempFiles != null) {
-                        for (Path generated : generatedTempFiles) {
-                            if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to
-                                // save the main file!
-                                logger.fine("(Will also try to permanently save generated thumbnail file "
-                                        + generated.toString() + ")");
-                                try {
-                                    // Files.copy(generated, Paths.get(dataset.getFileSystemDirectory().toString(),
-                                    // generated.getFileName().toString()));
-                                    int i = generated.toString().lastIndexOf("thumb");
-                                    if (i > 1) {
-                                        String extensionTag = generated.toString().substring(i);
-                                        dataAccess.savePathAsAux(generated, extensionTag);
-                                        logger.fine(
-                                                "Saved generated thumbnail as aux object. \"preview available\" status: "
-                                                + dataFile.isPreviewImageAvailable());
-                                    } else {
-                                        logger.warning(
-                                                "Generated thumbnail file name does not match the expected pattern: "
-                                                + generated.toString());
-                                    }
-
-                                } catch (IOException ioex) {
-                                    logger.warning("Failed to save generated file " + generated.toString());
-                                }
-                            }
-
-                            // ... but we definitely want to delete it:
-                            try {
-                                Files.delete(generated);
-                            } catch (IOException ioex) {
-                                logger.warning("Failed to delete generated file " + generated.toString());
-                            }
-                        }
-                    }
-                    // Any necessary post-processing:
-                    // performPostProcessingTasks(dataFile);
-                } else {
-                    // This is a direct upload 
-                    try {
-                        StorageIO<DvObject> dataAccess = DataAccess.getStorageIO(dataFile);
-                        //Populate metadata
-                        dataAccess.open(DataAccessOption.READ_ACCESS);
-                        //set file size
-                        logger.fine("Setting file size: " + dataAccess.getSize());
-                        dataFile.setFilesize(dataAccess.getSize());
-                        if (dataAccess instanceof S3AccessIO) {
-                            ((S3AccessIO<DvObject>) dataAccess).removeTempTag();
-                        }
-                    } catch (IOException ioex) {
-                        logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " ("
-                                + ioex.getMessage() + ")");
-                    }
-                    savedSuccess = true;
-                }
-
-                logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
-                boolean belowLimit = false;
-
-                try {
-                    //getting StorageIO may require knowing the owner (so this must come before owner is potentially set back to null
-                    belowLimit = dataFile.getStorageIO().isBelowIngestSizeLimit();
-                } catch (IOException e) {
-                    logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
-                }
-
-                if (savedSuccess && belowLimit) {
-                    // These are all brand new files, so they should all have
-                    // one filemetadata total. -- L.A.
-                    FileMetadata fileMetadata = dataFile.getFileMetadatas().get(0);
-                    String fileName = fileMetadata.getLabel();
+					// we may as well save them, by moving these generated images to the permanent
+					// dataset directory. We should also remember to delete any such files in the
+					// temp directory:
+					List<Path> generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()),
+							storageLocation);
+					if (generatedTempFiles != null) {
+						for (Path generated : generatedTempFiles) {
+							if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to
+												// save the main file!
+								logger.fine("(Will also try to permanently save generated thumbnail file "
+										+ generated.toString() + ")");
+								try {
+									// Files.copy(generated, Paths.get(dataset.getFileSystemDirectory().toString(),
+									// generated.getFileName().toString()));
+									int i = generated.toString().lastIndexOf("thumb");
+									if (i > 1) {
+										String extensionTag = generated.toString().substring(i);
+										dataAccess.savePathAsAux(generated, extensionTag);
+										logger.fine(
+												"Saved generated thumbnail as aux object. \"preview available\" status: "
+														+ dataFile.isPreviewImageAvailable());
+									} else {
+										logger.warning(
+												"Generated thumbnail file name does not match the expected pattern: "
+														+ generated.toString());
+									}
+
+								} catch (IOException ioex) {
+									logger.warning("Failed to save generated file " + generated.toString());
+								}
+							}
+
+							// ... but we definitely want to delete it:
+							try {
+								Files.delete(generated);
+							} catch (IOException ioex) {
+								logger.warning("Failed to delete generated file " + generated.toString());
+							}
+						}
+					}
+					// Any necessary post-processing:
+					// performPostProcessingTasks(dataFile);
+				} else {
+					try {
+						StorageIO<DvObject> dataAccess = DataAccess.getStorageIO(dataFile);
+						//Populate metadata
+						dataAccess.open(DataAccessOption.READ_ACCESS);
+						//set file size
+						logger.fine("Setting file size: " + dataAccess.getSize());
+						dataFile.setFilesize(dataAccess.getSize());
+						if(dataAccess instanceof S3AccessIO) {
+							  ((S3AccessIO<DvObject>)dataAccess).removeTempTag();
+						}
+					} catch (IOException ioex) {
+						logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " ("
+								+ ioex.getMessage() + ")");
+					}
+					savedSuccess = true;
+				}
 
-                    boolean metadataExtracted = false;
-                    boolean metadataExtractedFromNetcdf = false;
-                    if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
-                        /*
+				logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
+				boolean belowLimit = false;
+
+				try {
+					//getting StorageIO may require knowing the owner (so this must come before owner is potentially set back to null
+					belowLimit = dataFile.getStorageIO().isBelowIngestSizeLimit();
+				} catch (IOException e) {
+					logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
+				} 
+
+				if (savedSuccess && belowLimit) {
+					// These are all brand new files, so they should all have
+					// one filemetadata total. -- L.A.
+					FileMetadata fileMetadata = dataFile.getFileMetadatas().get(0);
+					String fileName = fileMetadata.getLabel();
+
+					boolean metadataExtracted = false;
+                                        boolean metadataExtractedFromNetcdf = false;
+					if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
+						/*
 						 * Note that we don't try to ingest the file right away - instead we mark it as
 						 * "scheduled for ingest", then at the end of the save process it will be queued
 						 * for async. ingest in the background. In the meantime, the file will be
 						 * ingested as a regular, non-tabular file, and appear as such to the user,
 						 * until the ingest job is finished with the Ingest Service.
-                         */
-                        dataFile.SetIngestScheduled();
-                    } else if (fileMetadataExtractable(dataFile)) {
-
-                        try {
-                            // FITS is the only type supported for metadata
-                            // extraction, as of now. -- L.A. 4.0
-                            // Note that extractMetadataNcml() is used for NetCDF/HDF5.
-                            dataFile.setContentType("application/fits");
-                            metadataExtracted = extractMetadata(tempFileLocation, dataFile, version);
-                        } catch (IOException mex) {
-                            logger.severe("Caught exception trying to extract indexable metadata from file "
-                                    + fileName + ",  " + mex.getMessage());
-                        }
-                        if (metadataExtracted) {
-                            logger.fine("Successfully extracted indexable metadata from file " + fileName);
-                        } else {
-                            logger.fine("Failed to extract indexable metadata from file " + fileName);
-                        }
-                    } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) {
-                        try {
-                            logger.fine("trying to extract metadata from netcdf");
-                            metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version);
-                        } catch (IOException ex) {
-                            logger.fine("could not extract metadata from netcdf: " + ex);
-                        }
-                        if (metadataExtractedFromNetcdf) {
-                            logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName);
-                        } else {
-                            logger.fine("Failed to extract indexable metadata from netcdf file " + fileName);
-                        }
+						 */
+						dataFile.SetIngestScheduled();
+					} else if (fileMetadataExtractable(dataFile)) {
+
+						try {
+							// FITS is the only type supported for metadata
+							// extraction, as of now. -- L.A. 4.0
+                                                        // Note that extractMetadataNcml() is used for NetCDF/HDF5.
+							dataFile.setContentType("application/fits");
+							metadataExtracted = extractMetadata(tempFileLocation, dataFile, version);
+						} catch (IOException mex) {
+							logger.severe("Caught exception trying to extract indexable metadata from file "
+									+ fileName + ",  " + mex.getMessage());
+						}
+						if (metadataExtracted) {
+							logger.fine("Successfully extracted indexable metadata from file " + fileName);
+						} else {
+							logger.fine("Failed to extract indexable metadata from file " + fileName);
+						}
+                                        } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) {
+                                            try {
+                                                logger.fine("trying to extract metadata from netcdf");
+                                                metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version);
+                                            } catch (IOException ex) {
+                                                logger.fine("could not extract metadata from netcdf: " + ex);
+                                            }
+                                            if (metadataExtractedFromNetcdf) {
+                                                logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName);
+                                            } else {
+                                                logger.fine("Failed to extract indexable metadata from netcdf file " + fileName);
+                                            }
 
-                    } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
+                                        } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
                         // Make sure no *uningested* tab-delimited files are saved with the type "text/tab-separated-values"!
                         // "text/tsv" should be used instead: 
                         dataFile.setContentType(FileUtil.MIME_TYPE_TSV);
                     }
-                }
+				}
                 if (unattached) {
                     dataFile.setOwner(null);
                 }
-                // ... and let's delete the main temp file if it exists:
-                if (tempLocationPath != null) {
-                    try {
-                        logger.fine("Will attempt to delete the temp file " + tempLocationPath.toString());
-                        Files.delete(tempLocationPath);
-                    } catch (IOException ex) {
-                        // (non-fatal - it's just a temp file.)
-                        logger.warning("Failed to delete temp file " + tempLocationPath.toString());
-                    }
-                }
-                if (savedSuccess) {
-                    // temp dbug line
-                    // System.out.println("ADDING FILE: " + fileName + "; for dataset: " +
-                    // dataset.getGlobalId());
-                    // Make sure the file is attached to the dataset and to the version, if this
-                    // hasn't been done yet:
-                    if (dataFile.getOwner() == null) {
-                        dataFile.setOwner(dataset);
-
-                        version.getFileMetadatas().add(dataFile.getFileMetadata());
-                        dataFile.getFileMetadata().setDatasetVersion(version);
-                        dataset.getFiles().add(dataFile);
-
-                        if (dataFile.getFileMetadata().getCategories() != null) {
-                            ListIterator<DataFileCategory> dfcIt = dataFile.getFileMetadata().getCategories()
-                                    .listIterator();
-
-                            while (dfcIt.hasNext()) {
-                                DataFileCategory dataFileCategory = dfcIt.next();
-
-                                if (dataFileCategory.getDataset() == null) {
-                                    DataFileCategory newCategory = dataset
-                                            .getCategoryByName(dataFileCategory.getName());
-                                    if (newCategory != null) {
-                                        newCategory.addFileMetadata(dataFile.getFileMetadata());
-                                        // dataFileCategory = newCategory;
-                                        dfcIt.set(newCategory);
-                                    } else {
-                                        dfcIt.remove();
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
+				// ... and let's delete the main temp file if it exists:
+				if(tempLocationPath!=null) {
+    				try {
+	    				logger.fine("Will attempt to delete the temp file " + tempLocationPath.toString());
+			    		Files.delete(tempLocationPath);
+				    } catch (IOException ex) {
+					    // (non-fatal - it's just a temp file.)
+    					logger.warning("Failed to delete temp file " + tempLocationPath.toString());
+	    			}				
+				}
+				if (savedSuccess) {
+					// temp dbug line
+					// System.out.println("ADDING FILE: " + fileName + "; for dataset: " +
+					// dataset.getGlobalId());
+					// Make sure the file is attached to the dataset and to the version, if this
+					// hasn't been done yet:
+					if (dataFile.getOwner() == null) {
+						dataFile.setOwner(dataset);
+
+						version.getFileMetadatas().add(dataFile.getFileMetadata());
+						dataFile.getFileMetadata().setDatasetVersion(version);
+						dataset.getFiles().add(dataFile);
+
+						if (dataFile.getFileMetadata().getCategories() != null) {
+							ListIterator<DataFileCategory> dfcIt = dataFile.getFileMetadata().getCategories()
+									.listIterator();
+
+							while (dfcIt.hasNext()) {
+								DataFileCategory dataFileCategory = dfcIt.next();
+
+								if (dataFileCategory.getDataset() == null) {
+									DataFileCategory newCategory = dataset
+											.getCategoryByName(dataFileCategory.getName());
+									if (newCategory != null) {
+										newCategory.addFileMetadata(dataFile.getFileMetadata());
+										// dataFileCategory = newCategory;
+										dfcIt.set(newCategory);
+									} else {
+										dfcIt.remove();
+									}
+								}
+							}
+						}
+					}
+				}
 
-                ret.add(dataFile);
-            }
-        }
+				ret.add(dataFile);
+			}
+		}
 
-        return ret;
-    }
+		return ret;
+	}
     
     public List<Path> listGeneratedTempFiles(Path tempDirectory, String baseName) {
         List<Path> generatedFiles = new ArrayList<>();
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index 4e27465f83f..d6d0be7a17b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -420,7 +420,6 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
         }
     }
 
-    @Asynchronous
     public void asyncIndexDatasetList(List<Dataset> datasets, boolean doNormalSolrDocCleanUp) {
         for(Dataset dataset : datasets) {
             asyncIndexDataset(dataset, true);

From a165efe0350a420409c802b3ae4400a20490c220 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 2 Nov 2023 12:42:19 -0400
Subject: [PATCH 662/922] (trying again) A few more improvements, borrowed from
 @qqmyers QDR fork: eliminates reindexing the parent dataverse when adding
 guestbooks, stops the update dataverse command from unnecessarily reindexing
 the underlying datasets in some cases, makes that reindex truly async. in the
 onSuccess part of the command. (#9635)

---
 .../harvard/iq/dataverse/GuestbookPage.java   | 19 +++++++-----
 .../command/impl/UpdateDataverseCommand.java  | 31 ++++++++++++++++---
 .../iq/dataverse/search/IndexServiceBean.java |  1 +
 3 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
index 9fb584a9133..5fe858ab636 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
@@ -288,19 +288,22 @@ public String save() {
            
         Command<Dataverse> cmd;
         try {
+            // Per recent #dv-tech conversation w/ Jim - copying the code 
+            // below from his QDR branch; the code that used to be here called
+            // UpdateDataverseCommand when saving new guestbooks, and that involved 
+            // an unnecessary reindexing of the dataverse (and, in some cases, 
+            // reindexing of the underlying datasets). - L.A.
             if (editMode == EditMode.CREATE || editMode == EditMode.CLONE ) {
                 guestbook.setCreateTime(new Timestamp(new Date().getTime()));
-                guestbook.setUsageCount(new Long(0));
+                guestbook.setUsageCount(Long.valueOf(0));
                 guestbook.setEnabled(true);
                 dataverse.getGuestbooks().add(guestbook);
-                cmd = new UpdateDataverseCommand(dataverse, null, null, dvRequestService.getDataverseRequest(), null);                
-                commandEngine.submit(cmd);
+                logger.info("Returned from command");
                 create = true;
-            } else {
-                cmd = new UpdateDataverseGuestbookCommand(dataverse, guestbook, dvRequestService.getDataverseRequest());
-                commandEngine.submit(cmd);
-            }
-
+            } 
+            cmd = new UpdateDataverseGuestbookCommand(dataverse, guestbook, dvRequestService.getDataverseRequest());
+            commandEngine.submit(cmd);
+        
         } catch (EJBException ex) {
             StringBuilder error = new StringBuilder();
             error.append(ex).append(" ");
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index 56c76f04c05..fe9415f39f9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -32,6 +32,8 @@ public class UpdateDataverseCommand extends AbstractCommand<Dataverse> {
 	private final List<DatasetFieldType> facetList;
         private final List<Dataverse> featuredDataverseList;
         private final List<DataverseFieldTypeInputLevel> inputLevelList;
+        
+        private boolean datasetsReindexRequired = false; 
 
 	public UpdateDataverseCommand(Dataverse editedDv, List<DatasetFieldType> facetList, List<Dataverse> featuredDataverseList, 
                     DataverseRequest aRequest,  List<DataverseFieldTypeInputLevel> inputLevelList ) {
@@ -74,9 +76,13 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
                 }
             }
             
-            DataverseType oldDvType = ctxt.dataverses().find(editedDv.getId()).getDataverseType();
-            String oldDvAlias = ctxt.dataverses().find(editedDv.getId()).getAlias();
-            String oldDvName = ctxt.dataverses().find(editedDv.getId()).getName();
+            Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
+            
+            DataverseType oldDvType = oldDv.getDataverseType();
+            String oldDvAlias = oldDv.getAlias();
+            String oldDvName = oldDv.getName();
+            oldDv = null; 
+            
             Dataverse result = ctxt.dataverses().save(editedDv);
             
             if ( facetList != null ) {
@@ -101,6 +107,14 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
                 }
             }
             
+            // We don't want to reindex the children datasets unnecessarily: 
+            // When these values are changed we need to reindex all children datasets
+            // This check is not recursive as all the values just report the immediate parent
+            if (!oldDvType.equals(editedDv.getDataverseType())
+                || !oldDvName.equals(editedDv.getName())
+                || !oldDvAlias.equals(editedDv.getAlias())) {
+                datasetsReindexRequired = true;
+            }
             
             return result;
 	}
@@ -110,9 +124,16 @@ public boolean onSuccess(CommandContext ctxt, Object r) {
         
         // first kick of async index of datasets
         // TODO: is this actually needed? Is there a better way to handle
+        // It appears that we at some point lost some extra logic here, where
+        // we only reindex the underlying datasets if one or more of the specific set
+        // of fields have been changed (since these values are included in the 
+        // indexed solr documents for dataasets). So I'm putting that back. -L.A.
         Dataverse result = (Dataverse) r;
-        List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
-        ctxt.index().asyncIndexDatasetList(datasets, true);
+        
+        if (datasetsReindexRequired) {
+            List<Dataset> datasets = ctxt.datasets().findByOwnerId(result.getId());
+            ctxt.index().asyncIndexDatasetList(datasets, true);
+        }
         
         return ctxt.dataverses().index((Dataverse) r);
     }  
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index d6d0be7a17b..4e27465f83f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -420,6 +420,7 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
         }
     }
 
+    @Asynchronous
     public void asyncIndexDatasetList(List<Dataset> datasets, boolean doNormalSolrDocCleanUp) {
         for(Dataset dataset : datasets) {
             asyncIndexDataset(dataset, true);

From eac4f10dfd11996d0e54c1d83941227e7efebb78 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva
 <142103991+jp-tosca@users.noreply.github.com>
Date: Thu, 2 Nov 2023 13:52:49 -0400
Subject: [PATCH 663/922] Update dev-usage.rst

Remove the instructions related to the re-deploy of containers in the background as of 11/02/2023 these instructions didn't work with the current containers.
---
 doc/sphinx-guides/source/container/dev-usage.rst | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 04c7eba7913..3a4426d5153 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -150,16 +150,6 @@ by recreating the container(s). In the future, more options may be added here.
 If you started your containers in foreground, just stop them and follow the steps for building and running again.
 The same goes for using Maven to start the containers in the background.
 
-In case of using Docker Compose and starting the containers in the background, you can use a workaround to only
-restart the application container:
-
-.. code-block::
-
-  # First rebuild the container (will complain about an image still in use, this is fine.)
-  mvn -Pct package
-  # Then re-create the container (will automatically restart the container for you)
-  docker compose -f docker-compose-dev.yml create dev_dataverse
-
 Using ``docker container inspect dev_dataverse | grep Image`` you can verify the changed checksums.
 
 Using a Debugger

From 3f59f7285b922326b6a4b49c04011eb275c05371 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Fri, 3 Nov 2023 10:31:41 -0400
Subject: [PATCH 664/922] Add the .gitattributes file to set LF by default on
 .sh files since these will run on docker.

---
 .gitattributes | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 .gitattributes

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000000..9860024f70a
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,4 @@
+# https://www.git-scm.com/docs/gitattributes
+
+# This set mandatory LF line endings for .sh files preventing from windows users to having to change the value of their git config --global core.autocrlf to 'false' or 'input'
+*.sh text eol=lf
\ No newline at end of file

From 9954c5edcc51c702351f39c8549b3e24170522e3 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 3 Nov 2023 12:27:28 -0400
Subject: [PATCH 665/922] checking in some intermediate changes/work in
 progress (#8549)

---
 .../iq/dataverse/EditDatafilesPage.java       |   2 +-
 .../dataverse/ingest/IngestServiceBean.java   | 511 ++++++++++--------
 2 files changed, 277 insertions(+), 236 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index a942830b19e..c40399e160f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -2169,7 +2169,7 @@ public void handleExternalUpload() {
                 - Max size specified in db: check too make sure file is within limits
             // ---------------------------- */
             /**
-             * @todo: this size check is probably redundant here, since the new
+             * @todo: this file size limit check is now redundant here, since the new
              * CreateNewFilesCommand is going to perform it (and the quota 
              * checks too, if enabled
              */
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 40dc3d6fdd6..d6496a97b97 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -30,6 +30,7 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileCategory;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
+import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.DataTable;
 import edu.harvard.iq.dataverse.DatasetField;
 import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
@@ -48,6 +49,8 @@
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.S3AccessIO;
 import edu.harvard.iq.dataverse.dataaccess.TabularSubsetGenerator;
+import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
 import edu.harvard.iq.dataverse.datavariable.SummaryStatistic;
 import edu.harvard.iq.dataverse.datavariable.DataVariable;
 import edu.harvard.iq.dataverse.ingest.metadataextraction.FileMetadataExtractor;
@@ -71,6 +74,7 @@
 import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.*;
+import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 
 import org.apache.commons.io.IOUtils;
 //import edu.harvard.iq.dvn.unf.*;
@@ -121,6 +125,7 @@
 import jakarta.jms.Message;
 import jakarta.faces.application.FacesMessage;
 import jakarta.ws.rs.core.MediaType;
+import java.text.MessageFormat;
 import ucar.nc2.NetcdfFile;
 import ucar.nc2.NetcdfFiles;
 
@@ -158,7 +163,8 @@ public class IngestServiceBean {
     private static String dateTimeFormat_ymdhmsS = "yyyy-MM-dd HH:mm:ss.SSS";
     private static String dateFormat_ymd = "yyyy-MM-dd";
     
-    // This method tries to permanently store new files on the filesystem. 
+    // This method tries to permanently store new files in storage (on the filesystem,
+    // in an S3 bucket, etc.).
     // Then it adds the files that *have been successfully saved* to the 
     // dataset (by attaching the DataFiles to the Dataset, and the corresponding
     // FileMetadatas to the DatasetVersion). It also tries to ensure that none 
@@ -167,56 +173,59 @@ public class IngestServiceBean {
     // DataFileCategory objects, if any were already assigned to the files). 
     // It must be called before we attempt to permanently save the files in 
     // the database by calling the Save command on the dataset and/or version.
+    
     public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
-                                                   List<DataFile> newFiles,
-                                                   DataFile fileToReplace,
-                                                   boolean tabIngest) {
-		List<DataFile> ret = new ArrayList<>();
-
-		if (newFiles != null && newFiles.size() > 0) {
-			// ret = new ArrayList<>();
-			// final check for duplicate file names;
-			// we tried to make the file names unique on upload, but then
-			// the user may have edited them on the "add files" page, and
-			// renamed FOOBAR-1.txt back to FOOBAR.txt...
+            List<DataFile> newFiles,
+            DataFile fileToReplace,
+            boolean tabIngest, 
+            UserStorageQuota quota) /*throws FileExceedsMaxSizeException, FileExceedsStorageQuotaException*/ {
+        List<DataFile> ret = new ArrayList<>();
+
+        if (newFiles != null && newFiles.size() > 0) {
+            // ret = new ArrayList<>();
+            // final check for duplicate file names;
+            // we tried to make the file names unique on upload, but then
+            // the user may have edited them on the "add files" page, and
+            // renamed FOOBAR-1.txt back to FOOBAR.txt...
             IngestUtil.checkForDuplicateFileNamesFinal(version, newFiles, fileToReplace);
-			Dataset dataset = version.getDataset();
-
-			for (DataFile dataFile : newFiles) {
-				boolean unattached = false;
-				boolean savedSuccess = false;
-				if (dataFile.getOwner() == null) {
-					unattached = true;
-					dataFile.setOwner(dataset);
-				}
-
-				String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier());
-				String driverType = DataAccess.getDriverType(storageInfo[0]);
-				String storageLocation = storageInfo[1];
-				String tempFileLocation = null;
-				Path tempLocationPath = null;
-				if (driverType.equals("tmp")) {  //"tmp" is the default if no prefix or the "tmp://" driver
-					tempFileLocation = FileUtil.getFilesTempDirectory() + "/" + storageLocation;
-
-					// Try to save the file in its permanent location:
-					tempLocationPath = Paths.get(tempFileLocation);
-					WritableByteChannel writeChannel = null;
-					FileChannel readChannel = null;
-
-					StorageIO<DataFile> dataAccess = null;
+            Dataset dataset = version.getDataset();
+
+            for (DataFile dataFile : newFiles) {
+                boolean unattached = false;
+                boolean savedSuccess = false;
+                if (dataFile.getOwner() == null) {
+                    unattached = true;
+                    dataFile.setOwner(dataset);
+                }
+                
+                String[] storageInfo = DataAccess.getDriverIdAndStorageLocation(dataFile.getStorageIdentifier());
+                String driverType = DataAccess.getDriverType(storageInfo[0]);
+                String storageLocation = storageInfo[1];
+                String tempFileLocation = null;
+                Path tempLocationPath = null;
+                long confirmedFileSize = 0L; 
+                if (driverType.equals("tmp")) {  //"tmp" is the default if no prefix or the "tmp://" driver
+                    tempFileLocation = FileUtil.getFilesTempDirectory() + "/" + storageLocation;
+
+                    // Try to save the file in its permanent location:
+                    tempLocationPath = Paths.get(tempFileLocation);
+                    WritableByteChannel writeChannel = null;
+                    FileChannel readChannel = null;
+
+                    StorageIO<DataFile> dataAccess = null;
 
-					try {
-						logger.fine("Attempting to create a new storageIO object for " + storageLocation);
-						dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
+                    try {
+                        logger.fine("Attempting to create a new storageIO object for " + storageLocation);
+                        dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
 
-						logger.fine("Successfully created a new storageIO object.");
-						/*
+                        logger.fine("Successfully created a new storageIO object.");
+                        /*
 						 * This commented-out code demonstrates how to copy bytes from a local
 						 * InputStream (or a readChannel) into the writable byte channel of a Dataverse
 						 * DataAccessIO object:
-						 */
+                         */
 
-						/*
+ /*
 						 * storageIO.open(DataAccessOption.WRITE_ACCESS);
 						 * 
 						 * writeChannel = storageIO.getWriteChannel(); readChannel = new
@@ -225,9 +234,9 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 						 * long bytesPerIteration = 16 * 1024; // 16K bytes long start = 0; while (
 						 * start < readChannel.size() ) { readChannel.transferTo(start,
 						 * bytesPerIteration, writeChannel); start += bytesPerIteration; }
-						 */
+                         */
 
-						/*
+ /*
 						 * But it's easier to use this convenience method from the DataAccessIO:
 						 * 
 						 * (if the underlying storage method for this file is local filesystem, the
@@ -235,214 +244,246 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 						 * 
 						 * Files.copy(tempLocationPath, storageIO.getFileSystemLocation(),
 						 * StandardCopyOption.REPLACE_EXISTING);
-						 */
-						dataAccess.savePath(tempLocationPath);
-
-						// Set filesize in bytes
-						//
-						dataFile.setFilesize(dataAccess.getSize());
-						savedSuccess = true;
-						logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
-
-                                            // TODO: reformat this file to remove the many tabs added in cc08330
-                                            extractMetadataNcml(dataFile, tempLocationPath);
-
-					} catch (IOException ioex) {
-                    logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")");
-					} finally {
-						if (readChannel != null) {
-							try {
-								readChannel.close();
-							} catch (IOException e) {
-							}
-						}
-						if (writeChannel != null) {
-							try {
-								writeChannel.close();
-							} catch (IOException e) {
-							}
-						}
-					}
+                         */
+                        dataAccess.savePath(tempLocationPath);
+
+                        // Set filesize in bytes
+                        //
+                        confirmedFileSize = dataAccess.getSize();
+                        dataFile.setFilesize(confirmedFileSize);
+                        savedSuccess = true;
+                        logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
+
+                        // TODO: reformat this file to remove the many tabs added in cc08330
+                        extractMetadataNcml(dataFile, tempLocationPath);
+
+                    } catch (IOException ioex) {
+                        logger.warning("Failed to save the file, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")");
+                    } finally {
+                        if (readChannel != null) {
+                            try {
+                                readChannel.close();
+                            } catch (IOException e) {
+                            }
+                        }
+                        if (writeChannel != null) {
+                            try {
+                                writeChannel.close();
+                            } catch (IOException e) {
+                            }
+                        }
+                    }
 
                     // Since we may have already spent some CPU cycles scaling down image thumbnails, 
-					// we may as well save them, by moving these generated images to the permanent
-					// dataset directory. We should also remember to delete any such files in the
-					// temp directory:
-					List<Path> generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()),
-							storageLocation);
-					if (generatedTempFiles != null) {
-						for (Path generated : generatedTempFiles) {
-							if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to
-												// save the main file!
-								logger.fine("(Will also try to permanently save generated thumbnail file "
-										+ generated.toString() + ")");
-								try {
-									// Files.copy(generated, Paths.get(dataset.getFileSystemDirectory().toString(),
-									// generated.getFileName().toString()));
-									int i = generated.toString().lastIndexOf("thumb");
-									if (i > 1) {
-										String extensionTag = generated.toString().substring(i);
-										dataAccess.savePathAsAux(generated, extensionTag);
-										logger.fine(
-												"Saved generated thumbnail as aux object. \"preview available\" status: "
-														+ dataFile.isPreviewImageAvailable());
-									} else {
-										logger.warning(
-												"Generated thumbnail file name does not match the expected pattern: "
-														+ generated.toString());
-									}
-
-								} catch (IOException ioex) {
-									logger.warning("Failed to save generated file " + generated.toString());
-								}
-							}
-
-							// ... but we definitely want to delete it:
-							try {
-								Files.delete(generated);
-							} catch (IOException ioex) {
-								logger.warning("Failed to delete generated file " + generated.toString());
-							}
-						}
-					}
-					// Any necessary post-processing:
-					// performPostProcessingTasks(dataFile);
-				} else {
-					try {
-						StorageIO<DvObject> dataAccess = DataAccess.getStorageIO(dataFile);
-						//Populate metadata
-						dataAccess.open(DataAccessOption.READ_ACCESS);
-						//set file size
-						logger.fine("Setting file size: " + dataAccess.getSize());
-						dataFile.setFilesize(dataAccess.getSize());
-						if(dataAccess instanceof S3AccessIO) {
-							  ((S3AccessIO<DvObject>)dataAccess).removeTempTag();
-						}
-					} catch (IOException ioex) {
-						logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " ("
-								+ ioex.getMessage() + ")");
-					}
-					savedSuccess = true;
-				}
+                    // we may as well save them, by moving these generated images to the permanent
+                    // dataset directory. We should also remember to delete any such files in the
+                    // temp directory:
+                    List<Path> generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()),
+                            storageLocation);
+                    if (generatedTempFiles != null) {
+                        for (Path generated : generatedTempFiles) {
+                            if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to
+                                // save the main file!
+                                logger.fine("(Will also try to permanently save generated thumbnail file "
+                                        + generated.toString() + ")");
+                                try {
+                                    // Files.copy(generated, Paths.get(dataset.getFileSystemDirectory().toString(),
+                                    // generated.getFileName().toString()));
+                                    int i = generated.toString().lastIndexOf("thumb");
+                                    if (i > 1) {
+                                        String extensionTag = generated.toString().substring(i);
+                                        dataAccess.savePathAsAux(generated, extensionTag);
+                                        logger.fine(
+                                                "Saved generated thumbnail as aux object. \"preview available\" status: "
+                                                + dataFile.isPreviewImageAvailable());
+                                    } else {
+                                        logger.warning(
+                                                "Generated thumbnail file name does not match the expected pattern: "
+                                                + generated.toString());
+                                    }
+
+                                } catch (IOException ioex) {
+                                    logger.warning("Failed to save generated file " + generated.toString());
+                                }
+                            }
+
+                            // ... but we definitely want to delete it:
+                            try {
+                                Files.delete(generated);
+                            } catch (IOException ioex) {
+                                logger.warning("Failed to delete generated file " + generated.toString());
+                            }
+                        }
+                    }
+                    // Any necessary post-processing:
+                    // performPostProcessingTasks(dataFile);
+                } else {
+                    // This is a direct upload 
+                    try {
+                        StorageIO<DvObject> dataAccess = DataAccess.getStorageIO(dataFile);
+                        //Populate metadata
+                        dataAccess.open(DataAccessOption.READ_ACCESS);
+                        
+                        confirmedFileSize = dataAccess.getSize();
+                        
+                        // For directly-uploaded files, we will perform the file size
+                        // limit and quota checks here. Perform them *again*, in 
+                        // some cases: a directly uploaded files have already been 
+                        // checked (for the sake of being able to reject the upload 
+                        // before the user clicks "save"). But in case of direct 
+                        // uploads via API, these checks haven't been performed yet, 
+                        // so, here's our chance.
+                        
+                        Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getEffectiveStorageDriverId());
+                        
+                        if (fileSizeLimit == null || confirmedFileSize < fileSizeLimit) {
+                        
+                            //set file size
+                            logger.fine("Setting file size: " + confirmedFileSize);
+                            dataFile.setFilesize(confirmedFileSize);
+                                                
+                            if (dataAccess instanceof S3AccessIO) {
+                                ((S3AccessIO<DvObject>) dataAccess).removeTempTag();
+                            }
+                            savedSuccess = true;
+                        }
+                    } catch (IOException ioex) {
+                        logger.warning("Failed to get file size, storage id, or failed to remove the temp tag on the saved S3 object" + dataFile.getStorageIdentifier() + " ("
+                                + ioex.getMessage() + ")");
+                    }
+                }
+                
+                // We will perform (another) quota check, and if still under quota
+                // (it's not impossible that somebody else may have uploaded more 
+                // stuff into the same collection/dataset etc., before this user 
+                // decided to click "save", for example!)
+        
+                if (systemConfig.isStorageQuotasEnforced() && quota != null) {
+                    long storageQuotaLimit = quota.getRemainingQuotaInBytes();
+                    if (confirmedFileSize > storageQuotaLimit) {
+                        savedSuccess = false; 
+                        //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(confirmedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                    }
+                }
+
+                logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
+                boolean belowLimit = false;
 
-				logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
-				boolean belowLimit = false;
-
-				try {
-					//getting StorageIO may require knowing the owner (so this must come before owner is potentially set back to null
-					belowLimit = dataFile.getStorageIO().isBelowIngestSizeLimit();
-				} catch (IOException e) {
-					logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
-				} 
-
-				if (savedSuccess && belowLimit) {
-					// These are all brand new files, so they should all have
-					// one filemetadata total. -- L.A.
-					FileMetadata fileMetadata = dataFile.getFileMetadatas().get(0);
-					String fileName = fileMetadata.getLabel();
-
-					boolean metadataExtracted = false;
-                                        boolean metadataExtractedFromNetcdf = false;
-					if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
-						/*
+                try {
+                    //getting StorageIO may require knowing the owner (so this must come before owner is potentially set back to null
+                    belowLimit = dataFile.getStorageIO().isBelowIngestSizeLimit();
+                } catch (IOException e) {
+                    logger.warning("Error getting ingest limit for file: " + dataFile.getIdentifier() + " : " + e.getMessage());
+                }
+
+                if (savedSuccess && belowLimit) {
+                    // These are all brand new files, so they should all have
+                    // one filemetadata total. -- L.A.
+                    FileMetadata fileMetadata = dataFile.getFileMetadatas().get(0);
+                    String fileName = fileMetadata.getLabel();
+
+                    boolean metadataExtracted = false;
+                    boolean metadataExtractedFromNetcdf = false;
+                    if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
+                        /*
 						 * Note that we don't try to ingest the file right away - instead we mark it as
 						 * "scheduled for ingest", then at the end of the save process it will be queued
 						 * for async. ingest in the background. In the meantime, the file will be
 						 * ingested as a regular, non-tabular file, and appear as such to the user,
 						 * until the ingest job is finished with the Ingest Service.
-						 */
-						dataFile.SetIngestScheduled();
-					} else if (fileMetadataExtractable(dataFile)) {
-
-						try {
-							// FITS is the only type supported for metadata
-							// extraction, as of now. -- L.A. 4.0
-                                                        // Note that extractMetadataNcml() is used for NetCDF/HDF5.
-							dataFile.setContentType("application/fits");
-							metadataExtracted = extractMetadata(tempFileLocation, dataFile, version);
-						} catch (IOException mex) {
-							logger.severe("Caught exception trying to extract indexable metadata from file "
-									+ fileName + ",  " + mex.getMessage());
-						}
-						if (metadataExtracted) {
-							logger.fine("Successfully extracted indexable metadata from file " + fileName);
-						} else {
-							logger.fine("Failed to extract indexable metadata from file " + fileName);
-						}
-                                        } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) {
-                                            try {
-                                                logger.fine("trying to extract metadata from netcdf");
-                                                metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version);
-                                            } catch (IOException ex) {
-                                                logger.fine("could not extract metadata from netcdf: " + ex);
-                                            }
-                                            if (metadataExtractedFromNetcdf) {
-                                                logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName);
-                                            } else {
-                                                logger.fine("Failed to extract indexable metadata from netcdf file " + fileName);
-                                            }
+                         */
+                        dataFile.SetIngestScheduled();
+                    } else if (fileMetadataExtractable(dataFile)) {
+
+                        try {
+                            // FITS is the only type supported for metadata
+                            // extraction, as of now. -- L.A. 4.0
+                            // Note that extractMetadataNcml() is used for NetCDF/HDF5.
+                            dataFile.setContentType("application/fits");
+                            metadataExtracted = extractMetadata(tempFileLocation, dataFile, version);
+                        } catch (IOException mex) {
+                            logger.severe("Caught exception trying to extract indexable metadata from file "
+                                    + fileName + ",  " + mex.getMessage());
+                        }
+                        if (metadataExtracted) {
+                            logger.fine("Successfully extracted indexable metadata from file " + fileName);
+                        } else {
+                            logger.fine("Failed to extract indexable metadata from file " + fileName);
+                        }
+                    } else if (fileMetadataExtractableFromNetcdf(dataFile, tempLocationPath)) {
+                        try {
+                            logger.fine("trying to extract metadata from netcdf");
+                            metadataExtractedFromNetcdf = extractMetadataFromNetcdf(tempFileLocation, dataFile, version);
+                        } catch (IOException ex) {
+                            logger.fine("could not extract metadata from netcdf: " + ex);
+                        }
+                        if (metadataExtractedFromNetcdf) {
+                            logger.fine("Successfully extracted indexable metadata from netcdf file " + fileName);
+                        } else {
+                            logger.fine("Failed to extract indexable metadata from netcdf file " + fileName);
+                        }
 
-                                        } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
+                    } else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
                         // Make sure no *uningested* tab-delimited files are saved with the type "text/tab-separated-values"!
                         // "text/tsv" should be used instead: 
                         dataFile.setContentType(FileUtil.MIME_TYPE_TSV);
                     }
-				}
+                }
                 if (unattached) {
                     dataFile.setOwner(null);
                 }
-				// ... and let's delete the main temp file if it exists:
-				if(tempLocationPath!=null) {
-    				try {
-	    				logger.fine("Will attempt to delete the temp file " + tempLocationPath.toString());
-			    		Files.delete(tempLocationPath);
-				    } catch (IOException ex) {
-					    // (non-fatal - it's just a temp file.)
-    					logger.warning("Failed to delete temp file " + tempLocationPath.toString());
-	    			}				
-				}
-				if (savedSuccess) {
-					// temp dbug line
-					// System.out.println("ADDING FILE: " + fileName + "; for dataset: " +
-					// dataset.getGlobalId());
-					// Make sure the file is attached to the dataset and to the version, if this
-					// hasn't been done yet:
-					if (dataFile.getOwner() == null) {
-						dataFile.setOwner(dataset);
-
-						version.getFileMetadatas().add(dataFile.getFileMetadata());
-						dataFile.getFileMetadata().setDatasetVersion(version);
-						dataset.getFiles().add(dataFile);
-
-						if (dataFile.getFileMetadata().getCategories() != null) {
-							ListIterator<DataFileCategory> dfcIt = dataFile.getFileMetadata().getCategories()
-									.listIterator();
-
-							while (dfcIt.hasNext()) {
-								DataFileCategory dataFileCategory = dfcIt.next();
-
-								if (dataFileCategory.getDataset() == null) {
-									DataFileCategory newCategory = dataset
-											.getCategoryByName(dataFileCategory.getName());
-									if (newCategory != null) {
-										newCategory.addFileMetadata(dataFile.getFileMetadata());
-										// dataFileCategory = newCategory;
-										dfcIt.set(newCategory);
-									} else {
-										dfcIt.remove();
-									}
-								}
-							}
-						}
-					}
-				}
+                // ... and let's delete the main temp file if it exists:
+                if (tempLocationPath != null) {
+                    try {
+                        logger.fine("Will attempt to delete the temp file " + tempLocationPath.toString());
+                        Files.delete(tempLocationPath);
+                    } catch (IOException ex) {
+                        // (non-fatal - it's just a temp file.)
+                        logger.warning("Failed to delete temp file " + tempLocationPath.toString());
+                    }
+                }
+                if (savedSuccess) {
+                    // temp dbug line
+                    // System.out.println("ADDING FILE: " + fileName + "; for dataset: " +
+                    // dataset.getGlobalId());
+                    // Make sure the file is attached to the dataset and to the version, if this
+                    // hasn't been done yet:
+                    if (dataFile.getOwner() == null) {
+                        dataFile.setOwner(dataset);
+
+                        version.getFileMetadatas().add(dataFile.getFileMetadata());
+                        dataFile.getFileMetadata().setDatasetVersion(version);
+                        dataset.getFiles().add(dataFile);
+
+                        if (dataFile.getFileMetadata().getCategories() != null) {
+                            ListIterator<DataFileCategory> dfcIt = dataFile.getFileMetadata().getCategories()
+                                    .listIterator();
+
+                            while (dfcIt.hasNext()) {
+                                DataFileCategory dataFileCategory = dfcIt.next();
+
+                                if (dataFileCategory.getDataset() == null) {
+                                    DataFileCategory newCategory = dataset
+                                            .getCategoryByName(dataFileCategory.getName());
+                                    if (newCategory != null) {
+                                        newCategory.addFileMetadata(dataFile.getFileMetadata());
+                                        // dataFileCategory = newCategory;
+                                        dfcIt.set(newCategory);
+                                    } else {
+                                        dfcIt.remove();
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
 
-				ret.add(dataFile);
-			}
-		}
+                ret.add(dataFile);
+            }
+        }
 
-		return ret;
-	}
+        return ret;
+    }
     
     public List<Path> listGeneratedTempFiles(Path tempDirectory, String baseName) {
         List<Path> generatedFiles = new ArrayList<>();

From 7892ec70b456e47a8a90fbbdef088a5605eba0a6 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Fri, 3 Nov 2023 13:08:46 -0400
Subject: [PATCH 666/922] Fix the missing deoployment on Payera detected on
 Windows caused by a race condition due the database not being ready when the
 DV container started.

---
 src/main/docker/scripts/init_3_wait_dataverse_db_host.sh | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 src/main/docker/scripts/init_3_wait_dataverse_db_host.sh

diff --git a/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh b/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh
new file mode 100644
index 00000000000..c234ad33307
--- /dev/null
+++ b/src/main/docker/scripts/init_3_wait_dataverse_db_host.sh
@@ -0,0 +1,4 @@
+#It was reported on 9949 that on the first launch of the containers Dataverse would not be deployed on payara
+#this was caused by a race condition due postgress not being ready. A solion for docker compose was prepared
+#but didn't work due a compatibility issue on the Maven pluggin [https://github.com/fabric8io/docker-maven-plugin/issues/888]
+wait-for "${DATAVERSE_DB_HOST:-postgres}:${DATAVERSE_DB_PORT:-5432}" -t 120
\ No newline at end of file

From 7fdfc65e6961fc9161501a4a7826d04325b9b137 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 3 Nov 2023 13:23:42 -0400
Subject: [PATCH 667/922] per latest feedback (#9635)

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 34 +++++++++++++------
 .../harvard/iq/dataverse/GuestbookPage.java   |  1 -
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index d79d0acfdb6..9b243f7b8c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -754,17 +754,26 @@ public boolean isIndexedVersion() {
         if (isIndexedVersion != null) {
             return isIndexedVersion;
         }
-        // The version is SUPPOSED to be indexed if it's the latest published version, or a
-        // draft. So if none of the above is true, we return false right away:
-
-        if (!(workingVersion.isDraft() || isThisLatestReleasedVersion())) {
+        
+        // Just like on the collection page, facets on the Dataset page can be
+        // disabled instance-wide by an admin:
+        if (settingsWrapper.isTrueForKey(SettingsServiceBean.Key.DisableSolrFacets, false)) {
             return isIndexedVersion = false;
         }
-
-        // ... but if it is the latest published version or a draft, we want to test
-        // and confirm that this version *has* actually been indexed and is searchable
-        // (and that solr is actually up and running!), by running a quick solr search:
-        return isIndexedVersion = isThisVersionSearchable();
+        
+        // The version is SUPPOSED to be indexed if it's the latest published version, or a
+        // draft. So if none of the above is true, we can return fasle.
+        // ... but if it is the latest published version or a draft, we want to 
+        // confirm that this version *has* actually been indexed. 
+        
+        if (workingVersion.isDraft()) {
+            return isIndexedVersion = workingVersion.getDataset().getIndexTime() != null;
+        } else if (isThisLatestReleasedVersion()) {
+            return isIndexedVersion = (workingVersion.getDataset().getIndexTime() != null)
+                    && workingVersion.getDataset().getIndexTime().after(workingVersion.getReleaseTime());
+        }
+        
+        return isIndexedVersion = false ;
     }
 
     /**
@@ -820,7 +829,11 @@ public List<FacetLabel> getFileTagsFacetLabels() {
     /**
      * Verifies that solr is running and that the version is indexed and searchable
      * @return boolean
-     */
+     * Commenting out this method for now, since we have decided it was not 
+     * necessary, to query solr just to figure out if we can query solr. We will
+     * rely solely on the latest-relesed status and the indexed timestamp from 
+     * the database for that. - L.A.
+     *
     public boolean isThisVersionSearchable() {
         // Just like on the collection page, facets on the Dataset page can be
         // disabled instance-wide by an admin:
@@ -862,6 +875,7 @@ public boolean isThisVersionSearchable() {
 
         return false;
     }
+    */
 
     /**
      * Finds the list of numeric datafile ids in the Version specified, by running
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
index 5fe858ab636..f54b1fb6117 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
@@ -298,7 +298,6 @@ public String save() {
                 guestbook.setUsageCount(Long.valueOf(0));
                 guestbook.setEnabled(true);
                 dataverse.getGuestbooks().add(guestbook);
-                logger.info("Returned from command");
                 create = true;
             } 
             cmd = new UpdateDataverseGuestbookCommand(dataverse, guestbook, dvRequestService.getDataverseRequest());

From c1afef9db9514567f064f336d7370d6f2173f74f Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 14:06:26 -0400
Subject: [PATCH 668/922] corrected function

---
 .../V6.0.0.3__10095-guestbook-at-request2.sql | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 src/main/resources/db/migration/V6.0.0.3__10095-guestbook-at-request2.sql

diff --git a/src/main/resources/db/migration/V6.0.0.3__10095-guestbook-at-request2.sql b/src/main/resources/db/migration/V6.0.0.3__10095-guestbook-at-request2.sql
new file mode 100644
index 00000000000..b6157e6a782
--- /dev/null
+++ b/src/main/resources/db/migration/V6.0.0.3__10095-guestbook-at-request2.sql
@@ -0,0 +1,34 @@
+-- This creates a function that ESTIMATES the size of the
+-- GuestbookResponse table (for the metrics display), instead
+-- of relying on straight "SELECT COUNT(*) ..."
+-- It uses statistics to estimate the number of guestbook entries
+-- and the fraction of them related to downloads,
+-- i.e. those that weren't created for 'AccessRequest' events.
+-- Significant potential savings for an active installation.
+-- See https://github.com/IQSS/dataverse/issues/8840 and 
+-- https://github.com/IQSS/dataverse/pull/8972 for more details
+
+CREATE OR REPLACE FUNCTION estimateGuestBookResponseTableSize()
+RETURNS bigint AS $$
+DECLARE
+  estimatedsize bigint;
+BEGIN
+  SELECT CASE WHEN relpages<10 THEN 0
+              ELSE ((reltuples / relpages)
+               * (pg_relation_size('public.guestbookresponse') / current_setting('block_size')::int))::bigint
+               * (SELECT CASE WHEN ((select count(*) from pg_stats where tablename='guestbookresponse') = 0 
+                   OR (select array_position(most_common_vals::text::text[], 'AccessRequest') 
+                       FROM pg_stats WHERE tablename='guestbookresponse' AND attname='eventtype') IS NULL) THEN 1
+                   ELSE 1 - (SELECT (most_common_freqs::text::text[])[array_position(most_common_vals::text::text[], 'AccessRequest')]::float
+                       FROM pg_stats WHERE tablename='guestbookresponse' and attname='eventtype') END)
+         END
+     FROM   pg_class
+     WHERE  oid = 'public.guestbookresponse'::regclass INTO estimatedsize;
+
+     if estimatedsize = 0 then
+     SELECT COUNT(id) FROM guestbookresponse WHERE eventtype!= 'AccessRequest' INTO estimatedsize;
+     END if;   
+
+  RETURN estimatedsize;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;

From cbf9774a2a10c014a02296799faef3868db857ff Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 3 Nov 2023 16:03:06 -0400
Subject: [PATCH 669/922] indextime stamps are not touched when Drafts are
 indexed. #9635

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 9b243f7b8c0..8040db7e3f6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -762,18 +762,21 @@ public boolean isIndexedVersion() {
         }
         
         // The version is SUPPOSED to be indexed if it's the latest published version, or a
-        // draft. So if none of the above is true, we can return fasle.
-        // ... but if it is the latest published version or a draft, we want to 
-        // confirm that this version *has* actually been indexed. 
+        // draft. So if none of the above is true, we can return false right away. 
+        if (!(workingVersion.isDraft() || isThisLatestReleasedVersion())) {
+            return isIndexedVersion = false;
+        }
+        // If this is the latest published version, we want to confirm that this 
+        // version was successfully indexed after the last publication 
         
-        if (workingVersion.isDraft()) {
-            return isIndexedVersion = workingVersion.getDataset().getIndexTime() != null;
-        } else if (isThisLatestReleasedVersion()) {
+        if (isThisLatestReleasedVersion()) {
             return isIndexedVersion = (workingVersion.getDataset().getIndexTime() != null)
                     && workingVersion.getDataset().getIndexTime().after(workingVersion.getReleaseTime());
         }
         
-        return isIndexedVersion = false ;
+        // Drafts don't have the indextime stamps set/incremented when indexed, 
+        // so we'll just assume it is indexed, and will then hope for the best.
+        return isIndexedVersion = true;
     }
 
     /**
@@ -987,10 +990,19 @@ public Set<Long> getFileIdsInVersionFromSolr(Long datasetVersionId, String patte
             logger.fine("Remote Solr Exception: " + ex.getLocalizedMessage());
             String msg = ex.getLocalizedMessage();
             if (msg.contains(SearchFields.FILE_DELETED)) {
+                // This is a backward compatibility hook put in place many versions
+                // ago, to accommodate instances running Solr with schemas that 
+                // don't include this flag yet. Running Solr with an up-to-date
+                // schema has been a hard requirement for a while now; should we 
+                // remove it at this point? - L.A. 
                 fileDeletedFlagNotIndexed = true;
+            } else {
+                isIndexedVersion = false;
+                return resultIds;
             }
         } catch (Exception ex) {
             logger.warning("Solr exception: " + ex.getLocalizedMessage());
+            isIndexedVersion = false; 
             return resultIds;
         }
 
@@ -1003,6 +1015,7 @@ public Set<Long> getFileIdsInVersionFromSolr(Long datasetVersionId, String patte
                 queryResponse = solrClientService.getSolrClient().query(solrQuery);
             } catch (Exception ex) {
                 logger.warning("Caught a Solr exception (again!): " + ex.getLocalizedMessage());
+                isIndexedVersion = false; 
                 return resultIds;
             }
         }

From 94b08c9f01fce8ac0cd9e77c2a38921c71257152 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:21:13 -0400
Subject: [PATCH 670/922] add user/name to privateURL token, refactor methods

---
 .../iq/dataverse/FileDownloadServiceBean.java | 25 +----------
 .../edu/harvard/iq/dataverse/FilePage.java    |  2 +-
 .../edu/harvard/iq/dataverse/api/Files.java   |  6 +--
 .../AuthenticationServiceBean.java            | 42 ++++++++++++++++++-
 4 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 55817d4a746..7a03f1a35dc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -4,7 +4,6 @@
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
@@ -16,8 +15,6 @@
 import edu.harvard.iq.dataverse.externaltools.ExternalToolHandler;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean;
 import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry;
-import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
-import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -75,8 +72,6 @@ public class FileDownloadServiceBean implements java.io.Serializable {
     @EJB
     AuthenticationServiceBean authService;
     @EJB
-    PrivateUrlServiceBean privateUrlService;
-    @EJB
     SettingsServiceBean settingsService;
     @EJB
     MailServiceBean mailService;
@@ -352,7 +347,7 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter
         User user = session.getUser();
         DatasetVersion version = fmd.getDatasetVersion();
         if (version.isDraft() || fmd.getDatasetVersion().isDeaccessioned() || (fmd.getDataFile().isRestricted()) || (FileUtil.isActivelyEmbargoed(fmd))) {
-            apiToken = getApiToken(user);
+            apiToken = authService.getValidApiTokenForUser(user);
         }
         DataFile dataFile = null;
         if (fmd != null) {
@@ -379,24 +374,6 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter
         }
     }
 
-    public ApiToken getApiToken(User user) {
-        ApiToken apiToken = null;
-        if (user instanceof AuthenticatedUser) {
-            AuthenticatedUser authenticatedUser = (AuthenticatedUser) user;
-            apiToken = authService.findApiTokenByUser(authenticatedUser);
-            if (apiToken == null || apiToken.isExpired()) {
-                //No un-expired token
-                apiToken = authService.generateApiTokenForUser(authenticatedUser);
-            }
-        } else if (user instanceof PrivateUrlUser) {
-            PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
-            PrivateUrl privateUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
-            apiToken = new ApiToken();
-            apiToken.setTokenString(privateUrl.getToken());
-        }
-        return apiToken;
-    }
-
     public void downloadDatasetCitationXML(Dataset dataset) {
         downloadCitationXML(null, dataset, false);
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
index bfae80ade27..479c8a429c6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java
@@ -1069,7 +1069,7 @@ public String preview(ExternalTool externalTool) {
         ApiToken apiToken = null;
         User user = session.getUser();
         if (fileMetadata.getDatasetVersion().isDraft() || fileMetadata.getDatasetVersion().isDeaccessioned() || (fileMetadata.getDataFile().isRestricted()) || (FileUtil.isActivelyEmbargoed(fileMetadata))) {
-            apiToken=fileDownloadService.getApiToken(user);
+            apiToken=authService.getValidApiTokenForUser(user);
         }
         if(externalTool == null){
             return "";
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
index ad24d81d996..d4fc92f912d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java
@@ -814,10 +814,8 @@ public Response getExternalToolFMParams(@Context ContainerRequestContext crc, @P
             return error(BAD_REQUEST, "External tool does not have file scope.");
         }
         ApiToken apiToken = null;
-        User u = getRequestUser(crc);
-        if (u instanceof AuthenticatedUser) {
-            apiToken = authSvc.findApiTokenByUser((AuthenticatedUser) u);
-        }
+        User user = getRequestUser(crc);
+        apiToken = authSvc.getValidApiTokenForUser(user);
         FileMetadata target = fileSvc.findFileMetadata(fmid);
         if (target == null) {
             return error(BAD_REQUEST, "FileMetadata not found.");
diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
index 496620cd6e8..848e57bc6b0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
@@ -21,10 +21,14 @@
 import edu.harvard.iq.dataverse.authorization.providers.shib.ShibAuthenticationProvider;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
+import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailData;
 import edu.harvard.iq.dataverse.confirmemail.ConfirmEmailServiceBean;
 import edu.harvard.iq.dataverse.passwordreset.PasswordResetData;
 import edu.harvard.iq.dataverse.passwordreset.PasswordResetServiceBean;
+import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
+import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean;
@@ -118,6 +122,9 @@ public class AuthenticationServiceBean {
     @EJB
     SavedSearchServiceBean savedSearchService;
 
+    @EJB
+    PrivateUrlServiceBean privateUrlService;
+ 
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
         
@@ -931,14 +938,45 @@ public List <WorkflowComment> getWorkflowCommentsByAuthenticatedUser(Authenticat
         return query.getResultList();
     }
 
-    public ApiToken getValidApiTokenForUser(AuthenticatedUser user) {
+    /**
+     * This method gets a valid api token for an AuthenticatedUser, creating a new
+     * token if one doesn't exist or if the token is expired.
+     * 
+     * @param user
+     * @return
+     */
+    public ApiToken getValidApiTokenForAuthenticatedUser(AuthenticatedUser user) {
         ApiToken apiToken = null;
         apiToken = findApiTokenByUser(user);
-        if ((apiToken == null) || (apiToken.getExpireTime().before(new Date()))) {
+        if ((apiToken == null) || apiToken.isExpired()) {
             logger.fine("Created apiToken for user: " + user.getIdentifier());
             apiToken = generateApiTokenForUser(user);
         }
         return apiToken;
     }
 
+    /**
+     *  Gets a token for an AuthenticatedUser or a PrivateUrlUser. It will create a
+     *  new token if needed for an AuthenticatedUser. Note that, for a PrivateUrlUser, this method creates a token
+     *  with a temporary AuthenticateUser that only has a userIdentifier - needed in generating signed Urls.
+     * @param user
+     * @return a token or null (i.e. if the user is not an AuthenticatedUser or PrivateUrlUser)
+     */
+
+    public ApiToken getValidApiTokenForUser(User user) {
+        ApiToken apiToken = null;
+        if (user instanceof AuthenticatedUser) {
+            apiToken = getValidApiTokenForAuthenticatedUser((AuthenticatedUser) user);
+        } else if (user instanceof PrivateUrlUser) {
+            PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
+            
+            PrivateUrl privateUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
+            apiToken = new ApiToken();
+            apiToken.setTokenString(privateUrl.getToken());
+            AuthenticatedUser au = new AuthenticatedUser();
+            au.setUserIdentifier(privateUrlUser.getIdentifier());
+            apiToken.setAuthenticatedUser(au);
+        }
+        return apiToken;
+    }
 }

From b4cf71e6b6a8970fed5faaaec2c258a2e24865e9 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:22:27 -0400
Subject: [PATCH 671/922] Fix potential infinite loop

---
 .../iq/dataverse/authorization/AuthenticationServiceBean.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
index 848e57bc6b0..1c0f5010059 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java
@@ -622,6 +622,7 @@ public AuthenticatedUser createAuthenticatedUser(UserRecordIdentifier userRecord
             String identifier = internalUserIdentifier + i;
             while ( identifierExists(identifier) ) {
                 i += 1;
+                identifier = internalUserIdentifier + i;
             }
             authenticatedUser.setUserIdentifier(identifier);
         } else {

From a7e4a7870ee85c3ba4d09396eb3a801b316f2c4c Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:24:45 -0400
Subject: [PATCH 672/922] use authSvc method to handle both auth and privateUrl
 cases

and get valid token if expired/missing (#10045)
---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 292aba0cee3..cbc0bcda6ac 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3930,10 +3930,7 @@ public Response getExternalToolDVParams(@Context ContainerRequestContext crc,
             }
             ApiToken apiToken = null;
             User u = getRequestUser(crc);
-            if (u instanceof AuthenticatedUser) {
-                apiToken = authSvc.findApiTokenByUser((AuthenticatedUser) u);
-            }
-            
+            apiToken = authSvc.getValidApiTokenForUser(u);
 
             ExternalToolHandler eth = new ExternalToolHandler(externalTool, target.getDataset(), apiToken, locale);
             return ok(eth.createPostBody(eth.getParams(JsonUtil.getJsonObject(externalTool.getToolParameters()))));

From 264448e98028f21dd9f61ec5f05e344044298315 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:28:38 -0400
Subject: [PATCH 673/922] handle urls signed for a privateUrlUser

---
 .../api/auth/SignedUrlAuthMechanism.java      | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java
index f8572144236..e2c2f2381d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java
@@ -3,7 +3,10 @@
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
+import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
+import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 
@@ -27,16 +30,18 @@ public class SignedUrlAuthMechanism implements AuthMechanism {
 
     @Inject
     protected AuthenticationServiceBean authSvc;
-
+    @Inject
+    protected PrivateUrlServiceBean privateUrlSvc;
+    
     @Override
     public User findUserFromRequest(ContainerRequestContext containerRequestContext) throws WrappedAuthErrorResponse {
         String signedUrlRequestParameter = getSignedUrlRequestParameter(containerRequestContext);
         if (signedUrlRequestParameter == null) {
             return null;
         }
-        AuthenticatedUser authUser = getAuthenticatedUserFromSignedUrl(containerRequestContext);
-        if (authUser != null) {
-            return authUser;
+        User user = getAuthenticatedUserFromSignedUrl(containerRequestContext);
+        if (user != null) {
+            return user;
         }
         throw new WrappedAuthErrorResponse(RESPONSE_MESSAGE_BAD_SIGNED_URL);
     }
@@ -45,8 +50,8 @@ private String getSignedUrlRequestParameter(ContainerRequestContext containerReq
         return containerRequestContext.getUriInfo().getQueryParameters().getFirst(SIGNED_URL_TOKEN);
     }
 
-    private AuthenticatedUser getAuthenticatedUserFromSignedUrl(ContainerRequestContext containerRequestContext) {
-        AuthenticatedUser authUser = null;
+    private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext containerRequestContext) {
+        User user = null;
         // The signedUrl contains a param telling which user this is supposed to be for.
         // We don't trust this. So we lookup that user, and get their API key, and use
         // that as a secret in validating the signedURL. If the signature can't be
@@ -54,17 +59,26 @@ private AuthenticatedUser getAuthenticatedUserFromSignedUrl(ContainerRequestCont
         // we reject the request.
         UriInfo uriInfo = containerRequestContext.getUriInfo();
         String userId = uriInfo.getQueryParameters().getFirst(SIGNED_URL_USER);
-        AuthenticatedUser targetUser = authSvc.getAuthenticatedUser(userId);
-        ApiToken userApiToken = authSvc.findApiTokenByUser(targetUser);
+        User targetUser = null; 
+        ApiToken userApiToken = null;
+        if(!userId.startsWith(PrivateUrlUser.PREFIX)) {
+        targetUser = authSvc.getAuthenticatedUser(userId);
+        userApiToken = authSvc.findApiTokenByUser((AuthenticatedUser)targetUser);
+        } else {
+            PrivateUrl privateUrl = privateUrlSvc.getPrivateUrlFromDatasetId(Long.parseLong(userId.substring(PrivateUrlUser.PREFIX.length())));
+            userApiToken = new ApiToken();
+            userApiToken.setTokenString(privateUrl.getToken());
+            targetUser = privateUrlSvc.getPrivateUrlUserFromToken(privateUrl.getToken());
+        }
         if (targetUser != null && userApiToken != null) {
             String signedUrl = URLDecoder.decode(uriInfo.getRequestUri().toString(), StandardCharsets.UTF_8);
             String requestMethod = containerRequestContext.getMethod();
             String signedUrlSigningKey = JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + userApiToken.getTokenString();
             boolean isSignedUrlValid = UrlSignerUtil.isValidUrl(signedUrl, userId, requestMethod, signedUrlSigningKey);
             if (isSignedUrlValid) {
-                authUser = targetUser;
+                user = targetUser;
             }
         }
-        return authUser;
+        return user;
     }
 }

From af490cdcb1b2b7ce713f67e93fe98028e596e9f2 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:31:42 -0400
Subject: [PATCH 674/922] use ! instead of # to avoid encoding issues in
 signedurls

---
 .../iq/dataverse/authorization/users/PrivateUrlUser.java        | 2 +-
 .../db/migration/V6.0.0.3__10093-privateurluser_id_update.sql   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 src/main/resources/db/migration/V6.0.0.3__10093-privateurluser_id_update.sql

diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java b/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java
index f64b5c301e7..03f018221fd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/authorization/users/PrivateUrlUser.java
@@ -12,7 +12,7 @@
  */
 public class PrivateUrlUser implements User {
 
-    public static final String PREFIX = "#";
+    public static final String PREFIX = "!";
 
     /**
      * In the future, this could probably be dvObjectId rather than datasetId,
diff --git a/src/main/resources/db/migration/V6.0.0.3__10093-privateurluser_id_update.sql b/src/main/resources/db/migration/V6.0.0.3__10093-privateurluser_id_update.sql
new file mode 100644
index 00000000000..260f191f557
--- /dev/null
+++ b/src/main/resources/db/migration/V6.0.0.3__10093-privateurluser_id_update.sql
@@ -0,0 +1 @@
+ update roleassignment set assigneeidentifier=replace(assigneeidentifier, '#','!') where assigneeidentifier like '#%';
\ No newline at end of file

From 9f04e07b451e5705b69c2707e49420338fc084da Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:37:32 -0400
Subject: [PATCH 675/922] fine logging

---
 .../harvard/iq/dataverse/externaltools/ExternalToolHandler.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
index cdde9fbe0e8..2fc4df808bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/externaltools/ExternalToolHandler.java
@@ -141,7 +141,7 @@ public String handleRequest(boolean preview) {
             if (requestMethod.equals(HttpMethod.POST)) {
                 String body = JsonUtil.prettyPrint(createPostBody(params).build());
                 try {
-                    logger.info("POST Body: " + body);
+                    logger.fine("POST Body: " + body);
                     return postFormData(body);
                 } catch (IOException | InterruptedException ex) {
                     Logger.getLogger(ExternalToolHandler.class.getName()).log(Level.SEVERE, null, ex);

From 204487d04879be88a1566dd454c5130e6d86bb8c Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 16:51:28 -0400
Subject: [PATCH 676/922] use authSvc methods, #10045 fix

---
 .../java/edu/harvard/iq/dataverse/DatasetPage.java   | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index fc18257196d..85a0277d39f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5910,14 +5910,7 @@ public void setFolderPresort(boolean folderPresort) {
     public void explore(ExternalTool externalTool) {
         ApiToken apiToken = null;
         User user = session.getUser();
-        if (user instanceof AuthenticatedUser) {
-            apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
-        } else if (user instanceof PrivateUrlUser) {
-            PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
-            PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
-            apiToken = new ApiToken();
-            apiToken.setTokenString(privUrl.getToken());
-        }
+        apiToken = authService.getValidApiTokenForUser(user);
         ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken, session.getLocaleCode());
         PrimeFaces.current().executeScript(externalToolHandler.getExploreScript());
     }
@@ -5925,8 +5918,9 @@ public void explore(ExternalTool externalTool) {
     public void configure(ExternalTool externalTool) {
         ApiToken apiToken = null;
         User user = session.getUser();
+        //Not enabled for PrivateUrlUsers (who wouldn't have write permissions anyway)
         if (user instanceof AuthenticatedUser) {
-            apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
+            apiToken = authService.getValidApiTokenForAuthenticatedUser((AuthenticatedUser) user);
         }
         ExternalToolHandler externalToolHandler = new ExternalToolHandler(externalTool, dataset, apiToken, session.getLocaleCode());
         PrimeFaces.current().executeScript(externalToolHandler.getConfigureScript());

From e08f5d79bdcf1351967d90bd832cf06ed5803364 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 17:35:47 -0400
Subject: [PATCH 677/922] release note

---
 doc/release-notes/10093-signedUrl_improvements.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/release-notes/10093-signedUrl_improvements.md

diff --git a/doc/release-notes/10093-signedUrl_improvements.md b/doc/release-notes/10093-signedUrl_improvements.md
new file mode 100644
index 00000000000..8f6ae89f981
--- /dev/null
+++ b/doc/release-notes/10093-signedUrl_improvements.md
@@ -0,0 +1,5 @@
+A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
+
+SignedUrls can now be used with PrivateUrl access, i.e. allowing PrivateUrl users to view Previews when they are configured to use signedUrls
+
+Launching a Dataset-level Configuration tool will automatically generate an api key when needed. This is consistent with how other types of tools work.

From cb5848ebf3729b5f86e2bca5b349b42080babf9e Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 17:41:16 -0400
Subject: [PATCH 678/922] doc update

---
 doc/sphinx-guides/source/api/external-tools.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst
index c72b52637c7..d802bc8714a 100644
--- a/doc/sphinx-guides/source/api/external-tools.rst
+++ b/doc/sphinx-guides/source/api/external-tools.rst
@@ -187,6 +187,7 @@ Using Example Manifests to Get Started
 ++++++++++++++++++++++++++++++++++++++
 
 Again, you can use :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` or :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as a starting point for your own manifest file.
+Additional working examples, including ones using signedUrls, are available at https://github.com/gdcc/dataverse-previewers .
 
 Testing Your External Tool
 --------------------------

From c208d04f1d5c7a325bcbf4628b550d9b58d713be Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 17:48:32 -0400
Subject: [PATCH 679/922] test updates

---
 .../harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java
index 8c9e0261bfa..da94b288bee 100644
--- a/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlUtilTest.java
@@ -102,7 +102,7 @@ public void testGetDatasetFromRoleAssignmentSuccess() {
         RoleAssignment ra = this.createTestRoleAssignment(dataset);
 
         assertNotNull(PrivateUrlUtil.getDatasetFromRoleAssignment(ra));
-        assertEquals("#42", ra.getAssigneeIdentifier());
+        assertEquals(PrivateUrlUser.PREFIX + "42", ra.getAssigneeIdentifier());
     }
 
     @Test
@@ -137,7 +137,7 @@ public void testGetDraftDatasetVersionFromRoleAssignmentSuccess() {
 
         DatasetVersion datasetVersionOut = PrivateUrlUtil.getDraftDatasetVersionFromRoleAssignment(ra);
         assertNotNull(datasetVersionOut);
-        assertEquals("#42", ra.getAssigneeIdentifier());
+        assertEquals(PrivateUrlUser.PREFIX + "42", ra.getAssigneeIdentifier());
     }
 
     @Test

From 98afbaf7057fa2f84cfa9fee9ac906a5eac4d48f Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 3 Nov 2023 17:52:42 -0400
Subject: [PATCH 680/922] another test

---
 .../edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java
index 1d054040e84..88f6a5bdbce 100644
--- a/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/util/json/JsonPrinterTest.java
@@ -114,7 +114,7 @@ public void testJson_RoleAssignment() {
         JsonObjectBuilder job = JsonPrinter.json(ra);
         assertNotNull(job);
         JsonObject jsonObject = job.build();
-        assertEquals("#42", jsonObject.getString("assignee"));
+        assertEquals(PrivateUrlUser.PREFIX + "42", jsonObject.getString("assignee"));
         assertEquals(123, jsonObject.getInt("definitionPointId"));
         assertEquals("e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getString("privateUrlToken"));
     }
@@ -135,7 +135,7 @@ public void testJson_PrivateUrl() {
         assertEquals("e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getString("token"));
         assertEquals("https://dataverse.example.edu/privateurl.xhtml?token=e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getString("link"));
         assertEquals("e1d53cf6-794a-457a-9709-7c07629a8267", jsonObject.getJsonObject("roleAssignment").getString("privateUrlToken"));
-        assertEquals("#42", jsonObject.getJsonObject("roleAssignment").getString("assignee"));
+        assertEquals(PrivateUrlUser.PREFIX + "42", jsonObject.getJsonObject("roleAssignment").getString("assignee"));
     }
 
     @Test

From f2eb91e8f7d4776d891684ff9a0dd05f71617860 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Sat, 4 Nov 2023 17:22:36 -0400
Subject: [PATCH 681/922] try longer lock wait

---
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e3a7fd0cfc3..1414dd32864 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -62,7 +62,7 @@ public class UtilIT {
     private static final String BUILTIN_USER_KEY = "burrito";
     private static final String EMPTY_STRING = "";
     public static final int MAXIMUM_INGEST_LOCK_DURATION = 15;
-    public static final int MAXIMUM_PUBLISH_LOCK_DURATION = 15;
+    public static final int MAXIMUM_PUBLISH_LOCK_DURATION = 20;
     public static final int MAXIMUM_IMPORT_DURATION = 1;
 
     private static SwordConfigurationImpl swordConfiguration = new SwordConfigurationImpl();

From d074c257c845678c75690e067a68d26daab0993d Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 6 Nov 2023 09:15:54 -0500
Subject: [PATCH 682/922] use prefixes in RoleAssigneeBean

---
 .../edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java
index 059d5a8ffd3..5429f5952dd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java
@@ -99,15 +99,15 @@ public RoleAssignee getRoleAssignee(String identifier, Boolean augmented) {
         switch (identifier.charAt(0)) {
             case ':':
                 return predefinedRoleAssignees.get(identifier);
-            case '@':
+            case AuthenticatedUser.IDENTIFIER_PREFIX:
                 if (!augmented){
                     return authSvc.getAuthenticatedUser(identifier.substring(1));
                 } else {
                     return authSvc.getAuthenticatedUserWithProvider(identifier.substring(1));
-                }                
-            case '&':
+                }
+            case Group.IDENTIFIER_PREFIX:
                 return groupSvc.getGroup(identifier.substring(1));
-            case '#':
+            case PrivateUrlUser.PREFIX:
                 return PrivateUrlUtil.identifier2roleAssignee(identifier);
             default:
                 throw new IllegalArgumentException("Unsupported assignee identifier '" + identifier + "'");

From 780a0d5ae70bbd5e83ce843b36800304dbb46a0e Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Mon, 6 Nov 2023 09:24:27 -0500
Subject: [PATCH 683/922] use string, add missing import

---
 .../edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java    | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java
index 5429f5952dd..88acc1916cf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/RoleAssigneeServiceBean.java
@@ -11,6 +11,7 @@
 import edu.harvard.iq.dataverse.authorization.groups.impl.explicit.ExplicitGroupServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.GuestUser;
+import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.mydata.MyDataFilterParams;
 import edu.harvard.iq.dataverse.privateurl.PrivateUrlUtil;
@@ -96,8 +97,8 @@ public RoleAssignee getRoleAssignee(String identifier, Boolean augmented) {
         if (identifier == null || identifier.isEmpty()) {
             throw new IllegalArgumentException("Identifier cannot be null or empty string.");
         }
-        switch (identifier.charAt(0)) {
-            case ':':
+        switch (identifier.substring(0,1)) {
+            case ":":
                 return predefinedRoleAssignees.get(identifier);
             case AuthenticatedUser.IDENTIFIER_PREFIX:
                 if (!augmented){

From 62b85b6229dfde7baba0395b3ac2362b3e96e9ac Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 6 Nov 2023 13:49:59 -0500
Subject: [PATCH 684/922] add release note #9590

---
 doc/release-notes/9590-intellij-redeploy.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/9590-intellij-redeploy.md

diff --git a/doc/release-notes/9590-intellij-redeploy.md b/doc/release-notes/9590-intellij-redeploy.md
new file mode 100644
index 00000000000..4c6ab29ea9f
--- /dev/null
+++ b/doc/release-notes/9590-intellij-redeploy.md
@@ -0,0 +1,3 @@
+Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
+
+For details, see https://dataverse-guide--10088.org.readthedocs.build/en/10088/container/dev-usage.html

From 334beb8beef6965bd94e6cfdc6955a4986853b87 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 6 Nov 2023 13:57:47 -0500
Subject: [PATCH 685/922] explain fast redeploy with IntelliJ #9590

---
 .../source/container/dev-usage.rst            | 44 ++++++++++++++++---
 1 file changed, 37 insertions(+), 7 deletions(-)

diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 3a4426d5153..b2547306b03 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -141,16 +141,46 @@ Alternatives:
   Options are the same.
 
 
-Re-Deploying
-------------
+Redeploying
+-----------
+
+Rebuild and Running Images
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The safest way to redeploy code is to stop the running containers (with Ctrl-c if you started them in the foreground) and then build and run them again with ``mvn -Pct clean package docker:run``.
+
+IntelliJ IDEA Ultimate and Payara Platform Tools
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have IntelliJ IDEA Ultimate (note that `free educational licenses <https://www.jetbrains.com/community/education/>`_ are available), you can install `Payara Platform Tools <https://plugins.jetbrains.com/plugin/15114-payara-platform-tools>`_ which can dramatically improve your feedback loop when iterating on code.
+
+The following steps are suggested:
+
+- Go to the Payara admin console (either at https://localhost:4848 or http://localhost:4849) and undeploy the dataverse application under "Applications".
+- Install Payara Platform Tools.
+- Under "Server":
+
+  - Click "Run" then "Edit Configurations".
+  - Click the plus sign and scroll down to Payara Server and click "Remote".
+  - For "Name" put "Payara in Docker" or something reasonable.
+  - Under "Application server" select a local directory that has the same version of Payara used in the container. This should match the version of Payara mentioned in the Installation Guide under :ref:`payara`.
+  - Change "Admin Server Port" to 4849.
+  - For username, put "admin".
+  - For password, put "admin".
+
+- Under "Deployment":
+
+  - Click the plus button and clien "Artifact" then "dataverse:war".
 
-Currently, the only safe and tested way to re-deploy the Dataverse application after you applied code changes is
-by recreating the container(s). In the future, more options may be added here.
+- Under "Startup/Connection":
 
-If you started your containers in foreground, just stop them and follow the steps for building and running again.
-The same goes for using Maven to start the containers in the background.
+  - Click "Debug" and change the port to 9009.
 
-Using ``docker container inspect dev_dataverse | grep Image`` you can verify the changed checksums.
+- Click "Run" and then "Debug Payara in Docker". This initial deployment will take some time.
+- Go to http://localhost:8080/api/info/version and make sure the API is responding.
+- Edit ``Info.java`` and make a small change to the ``/api/info/version`` code.
+- Click "Run" then "Debugging Actions" then "Reload Changed Classes". The deployment should only take a few seconds.
+- Go to http://localhost:8080/api/info/version and verify the change you made.
 
 Using a Debugger
 ----------------

From d966ce1daa41e3ab3a155e2fde91e3fb00ab1b1d Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 6 Nov 2023 13:59:24 -0500
Subject: [PATCH 686/922] remove unused image #9590

---
 modules/nginx/img.png | Bin 71929 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 modules/nginx/img.png

diff --git a/modules/nginx/img.png b/modules/nginx/img.png
deleted file mode 100644
index 278ee490b2d6d7c61b641cae4133b022c66098e9..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 71929
zcmeF3Wmr_-7w=I}P+C-aXb`1429O335a}E`q@-s^B?rj?5hVr@l$7q2kcOc<q`QXh
zd&b{C?!7PWeRp5oc^=<z<{Z!Lv-jF-eZJpy!e6N=65gk}kA;OrsH`OS8Vl<-3iy|X
zcNh2z;*#Y#7S=l~WjSdrsL9sMohm2o5$?A;Y~S>run*2WA$$?`;>k?vhj)xKEK%-W
ziSAw$0&sWX+k?K}1P;!VPolTYQ|nJ)>brh@6sUuM<BI})i)hiYlpE1ey<O0Dz|Bd3
z3L`>^`9ZiGBVrA7b3WGC6-MW;XX3s7ozFm=_uq$<2=JK2I^F*L)d7Y$Fx%gAfKzE1
z|Mvq-6uXT#{%5OFH=@6eND0M`<m>-_nF67N20y@NCmRRt)u-CnKCh?q-IFH2=TIRk
zvYjjI*HtkTDp;1Yy{0bir$k!uW^$A6=2Q{3TYqx?P~&@dvU7v}tpCNf<agb_H^M}v
zj8rtQJ6(ua{$hh%4@}c_@w1J4?ID5gU|J3)Q+why3$uZ(rg}B>uau%B4NZKu%h9_5
zH?>TBPf>0t30*bu?K0Qp1YIKG*OYR{oos0wUUPn4)hDw)c~@%^Ws@-t#?bkY<qoEp
z{A2IK0mW|TfQu>D^Ytt*%QUk)S8{7DQ<o==H}zXJn+417u!*vny=^Dx|Bl&pG&72c
zouv5cWUldYfxIz$+%jIZYbDv)<apW*^@1pfqk^vY<alOIs7bb_Pjoe<r1}1{kH9ai
z1`bP<*Q*N8lvSK{L#|64%13mTPZwhfEz4A=N62gB{7&Wq0`&ckM!KD9w{>RyPG+3y
z#~dYmYDV*GYY#gPHu4HTWyDC<IFt=2SOLF|E3csD&4hUbXV=jfWbS8QLHXuJpN^y9
z4lpKG3EDb29&_$Ps>|<jK~Jcis?0=>$1NKB+mVMm1;pF-y}ORIlgC6xF25fw4{NG(
za@Xz2une@_&zZAIH148yuH${)IJwAM<e3cfexZVyN8hCAI}eoZ<Akh|+>_Y*h20&_
zhSc+%g{_vfQQ2#mcVmC}MXvzrRjjU@_0BU{+&@YkBaM}xzr)>li6-xeL0#<BI_m>Z
zDlhS@V#+CR>1}DR(6ca*w=K)2w9?7AMGX3UvoPTO^Zih!HyeX@(U+BtS3mPh`^1*L
z8ZV4{Tigg$n<q!fI*(_L`c+IoT4R1O4P}mZ6lq-KMi7o9R~gY_x%rLP=j9OL>9?|p
zx}W^rVb^D?_QnmBj#->(M-l$g0tsg$h5p*akHCHDm8e69$=_Z_7qeT5#b{&ExshF;
z!)IH6?mZvQcuDh|=7CIM|IFvDs>Nt4;6df#=ItFHG*b^1Rf`SBn6`uA4T4i1sx@a1
zaAS&SB@&3ISi;pXqZA+;w~Ba(M}Xo)`_B2DB&bVWl|4wF@tUiT%apoWS0CA|UjAhG
zNmEVqdxX(FwsGrZ$1XB;u!i!;tk15o%c#E1NL}OgQDL_-TdLMQ{a9WTEvw<1#Kz0-
zaj#-7uGm0FjFw6no`uybN$3GZx&&Pl*moMwakB(Vl_}@Ot2dv+1eRbk9%Dai(4p5W
z!i_nzn^Kqa#Kx9{ymP>`>Kam)GVZnw@^7XiF~&LBYauQ?2mV~KTzAlyw{!(edOz|e
zZQ|P}{j$W`-|Cv~Qh%)7smb%b+;1=MQ{k%4%&%JbpdCZMZj<Kc>4@7oDv5HBeHH8z
zM(dgFecnA3&@YXvO+Datv0YhsHAqyr^Rsf+0OY&ZWDw(Dl2CW9;5>$sy15jKE+{Un
znOHytT)T}RdZA&&lBa)inNT;QhR(*(qAQ;b+Qy66nzzIaORB|!o*aa3O3fT@_VA<f
z7Z?95{;>iT<#>-bLvs@SmNRf`?Z%o~q~+!&QoW2D*SL)2(>^A8Hr*k2r@j3*I<0X7
z(JAhmDpKlfDgFf&EF*-O-pSl45#oRLB9@Vrk~BX072{VW5@*4|5Z?k|_Li$H8K&Zi
z>~g%`5Puy8#;2Uu#$i5RidW})_Ot9Y%jb@VuftznJTsD^1uug0H{^H_qT5Ukl7;1?
z27&@O2wA!HCi^_&tARvWBGJsa>cwdBf^O`-AkL3+{+Wc71n8H-!AK{fdj%o0br~{B
z<IP_4?CuH<<t-U%+>WJmKBcs-a44FmG{jYj6`c!d78swdqQLe(@8jAU8aAXoRA{5s
zi`5Lip0qFg$$YclPFHC04FWH6L8M7c^nD(HOt}w5%XTN(7wF#umqb_L7Y>l0O<+&_
z$Z9BQ(Nvev-qh@C>d?zU*;H6!Z+n~56DkWif7F;IyLT3qEE|4Jp+1&8djV`4O3deC
zap0ItJapvNR{5Ao0uKX+(j#JqLHBlAFD=)w)4MPQEd&zSR;uD`GAS8k7)e!U6_%pe
zPJjTE$4s=pM0!LEvyZp2)P2c0s7O3J+K*klnDCv+Ri4f9Aeb?F5MhRN-nNDK0pVJc
z1rj51tjB^1xLl;OuN_1PA~iDk>UIu?xrJ=wY&?wJ2jsPdL7H<dq0*Nvn$dWSpTR|G
z0oSnVOLVt=a<dMJuHSR|`n8V7;GgaaWi8avdU@^Ap|c$seBkyrIkCWPENN$t0cr+$
zWXN36kjI#@Zf7YQj#7dDK-0Xu|E!&-e21}O$%KRnV}g^>M2wSCDepd-tv>jusL_{5
zU=``(lo-_v0`Il`g4*dy(KM{F)3o1uS0C(e<T0Wx$TfTG_xrHf-nR+Im#TlGWg#OF
zEkCfr6y70}mqNoxC+;nTUNSgIBN5rGFdd?FvX&3|3`z5<+PLbrtzC>hGYA_TttVKF
zolL&K;?Xq@q{m(eh3|VazJ(jt@xb%Xj_|!1vSqMhmCq0|-g33gYJsQgl~sn@5p(i<
zFu5tD4bCH8hYb$Gb-A4-MLywK=py%UcqC)kLdbGDb*hmK{uRNl*n{p+X(ZD$LkB;K
zN2W75sz?mWR<?F)si>bB)hr*Wc$lY*{LZvRs+sRtvuA0aOpcu0u(F5IBt_mDESLd)
z&0)xU5dl%2enZNu4_PGh`kv4ME@KkAML09l(8{OFHq!l3a!+{i-^}Bm9pi)b_+96M
z#1U|y;ttdfrm(mynfcTli6|0+tw_uH$Nm7W8b_kY+^u5TVH7(KhecNzG2HHW<Qj>`
zlpv!ND~z*W7;t6J@~7kJ?RI+_vpaQlCX<@ZBpOQ-P48^J?6rp4Q0)(o?SN6ucaxU%
z>6WC?u`lG*rJi33m4uI@7sQ})koAYH<oj|CLEs^}^cYzgwk1(8%3<ahtU`<`g}Qn2
z5Cz*v;`zpdhxF;*SAN-161q4@iO|q3P(_m7RIGlH`2AblR0L-rVmp#9`9rC79zAN`
zJl)+>FwuxVEm9(;E{kYGd@+)L18j$y`)B1m#jG4);x65t-$zrHM8X+_l?VgFwUiG^
zXk|j=2qVGxQ*Q~I28o=D+?ObM75ZYCk#@5NRz|W^w|EsEAPnR*L~w6+TznvTAsbDq
zKC5=N&IQIPmKQ<@a@R8Sfx8;=!a#LNXr?^UgM7$#5#iTb@3m|Y18bs*x81gQAe-f_
z&Xq4ZS^al1$weJN%&nrtT(HMtOzN#~gpYKxcx9uR9?C_Q`uhyLh9TE?I!eW0S+Flr
z8RQqiuTGt|5H@FwOTnwNW^K2c#!JKp3%J{4X-#equR?R1kZtvB==_Gf!iJjYMpq)&
zl@J~x%R0)Z@qZsWMDgziNZK4S&G53$r^Ap$AEXLWWX89X>40LBiZ2P9P#kXJ;~B-;
zG{*k<4Yo`sCpD7^9wx4v>_`*4N{CKj%Brh@Kl+x#;rcY@m_#l(Q4O5Y;8PaUGpH*8
z{}y}C68{UCLOM9`vE1XFAGpn{Eb>TeJ0;;=dvOQswN~-y%RxhluO>J7EdCo&!a^q5
zAGx?0Iaxo?5|N(E+*s=o4AL%%`d^53xE+isFKLjA_Z#|c{Y)czeE8=UF}Eaz1up37
z1NmK2e@i2}_2rsGK7{cxA8zacTsOM^(PG)OYq$1ohq}d(rqfQ*8rx@E76l@o*EcyU
zr(aJArUinT4#7hmZkLmIsS^oYbu$a;{0>Z*%<<|TCaI;*lX(zp%6wmWK19w421U^9
z=Qp#&n#7#^CV58I$g;&N*{VL&nE_^mzhuw=f87`FkRc8>g*Np=<X$K5qts-C(_r)n
ziN@jQWBsJn+MlA8eiV^4zIfkyX8^5RQv$9>4;UH_!pdNF<%N#h^bv4(uN&7S$!4;$
zmV+Mgyt#DEB{P-ty@og3TZ|IT>VYYxA>czTVq#(A5$eXU>7Jma`*~k0!G{VjNME%|
zI&?hiY1w*}StkaIT_-szmOUJRcn=b$$G9BNh$u=nX1^z3syVa}4|}rRh22pB0S`3n
zXT)rp1$fB$wFykdqE5evcXfBG^@{$@o#g_a19O7{t{gjgTu#q92n{rjXus^`8%&a7
zVU2MggT})kEg3XN^4|1#I_l^0*Hs`fW#@{q<5vo2N)i6=U*VhRn7Dz|8ScwGYE=`S
zdh^XLqC)M~wj3y+w$w4cOvsVY(MRYY+mIp%``)hmC6@p(&jZ@!Ng=z@3FGALhTkG5
znE@XiH>t*&C3}Tx?1j4&sp}`W{@&YlATuhw={9%Hsm(m~J_LITrtom>!G5F6i%WE{
zo*`&bC}&IMt1x$A7M6M1`Zme0iz(J(OmU^etUv%1O?Rp1<+oH?Mf;B#E9<}cJ-X7~
z{pa8#X4v1v5c5qjxwzH8f5oKJvj6|U|7}P>dq9H7|M&U$M;teNQeqAziUWILNAv-8
zkiky<$*e1oW3yngya8v^UGKH@ur8n7df{K!n6J_(BWrJ=eNhr8A2C@nm*n5abx2-!
zeZIN8Q@dM`Hg#dhh@fy~hV9(#D2=a0vH$&CDNA1NwQOL$K&L1XP-6~CBRRqUId=Ld
z$X^w?-2{}v*I2yM5)9~yO=afLN5G8*G5>RkEA4ZBSj9~Prh-)uIM=M+Z=>!8DnEf&
z(?Hp}93?!fbUXF*_JZl1?J@@WxK*dvB~cdO^S$v!01N7P7E?hzSO8M&in_CvWZL=b
zi(PK7X_xlrod=lO0F(G@pH8dMCA>E+=`NyAa<pAsjFBpsmv8HVxCsre@fH<t;SgzY
z*B`$23)lvV4_bOZC7zC@iQrAA!S%OZV;4+u;H*sGbEvzHTB017cL6=rI1Z(DqWIdt
zN>a9(7SLb>TTWO;cXJmmO_wX#lm^^f%wd0|e)O&uQ~Oy}&UogT+%T~8Z%vZXX`R>?
z*6Tz|p0AaRnfQ$CAD;!{b7g0qVG3_WS`RJQ(TFatN(}wXK-OkqV*{wIjPi((&|Fe!
zNk0gvkKE<m;;N%(q^?gFW#w~7quwJ{vyk4oM4{6D3*l^Ut{b>mv&qiCesB?*J-^p>
z7OyU8XG84(+bVAF0`P#S*5*rPWo60)D=j9}`2;nrB4)ydHO&{zcQmZ2`%-K@b2&3!
zEw@4@S50I-*mb*n%v0qnZkzq&`u1*pdSzYx_j2ARu&Fb|%(H^3Z#_IM1S1i2CVx7F
zpLldf5X`xDbIwf0wVw$$o~csW=NESYY(eY#`lK;2?dEcI8K_YmX;m!tQg?l`T!VyA
z4r3#su)RL<-QoR1oR%THgWG!*Yu!L)HlyKJZx-88s@VjLvYz!9t4}XzT7DG(oTSMr
zoaNJL;DFhyhg+P^&UUiYlN(Xs!kUfT<$&wefG&s9ULETcmo~m$fuWC%l`|D;zZ#xC
zE39*!00_ZUBJR@w04H<-SVYe`7Z{J~mx3c&RzMxe$%cfQpKX<*CMR(9)g+FF`IbLw
zj%u2}06tx@Dsihfw{IOJk3j%7)O|eV4557vRO+rQP6f(L2ff0O372--&DH+j$Y^*@
z<F=|cw|Fgx;ntvWywu>-IZjggeM#YP^8=sS8LX>sS4uHFaL*c)d!*!<O1tOh&@>T;
zPtK!b1mA6cNC!8ydeFGsT%GKeSZS_e>fs)VlW8j;+s|H~TpgG79%<2xbbH%YB0bLQ
zFiFc0Q$xA~3|KuFpgjq-pA~Nrp)iwv8DKPqy$3M?ml=6dK|#{2DEOjnMwCvc9!0mZ
zIk3n%fU3i%GCp0jir^78RXV8mR`l+G_0ZdEA;MMu)Jbn5_O$_O7zTiK10=-+>9bS4
z5hbx(cd)^}0o1|HHni60v(>ap?xUv>Ts5|1oldV><v=zmE*ss{YEJ&|=Dr;ST%&H-
zF~~yv;`#TVCzLiR8pQOD=;^+(>)B7~(o34JU%ytktBk&c{CO-q=j%pk5y`vZ3?Wk&
zLkOndrg;xw6_>L%;RcO{i<#X|-#;1Tddp7qgWB~iV+;T?Vn4397maxy2@Z>L_*_+{
zg#3G`5&z@TG;N!B`^gc1L$>MF%L5em0ZJ<1rw;%U@@jGnC-rSowO3NTX8DF*@+Sz6
z8I?ZhWPSgGy6|ZXr{br&jR^oTylE_BPvyc87?x7>{2N@Fx6iM*ZUEi_AtozO^f#lP
zybW8Y-z5^?dUa3a>N!<d(qXUAR6a;4b5Q|uF*(+C(r{9bAz4~+R93ME2|5NjKh*g;
znY9C4IvX#Sxx0ZX6moLL3KLSR8V4fY&>Kzg3P5?NfBjgADk3;0Qg<V<W<Ce1`|X;Q
zLLumQm^hI1w}X7PT$d1vqkX)WyZ$G&d}Bs#9thJI*%U4cKMZuq5{FeUQMxx?9p_=&
z2Cf?s#PN|sd|k-Xt+RhWHq58mVUs#j7PKUBb{_X0?jLni8n_3W1%Oz?TgPtBN{$GQ
z5~Zt1twMkyl?SMmg2=x-#6wc=0fqd|QyV_c>ZQ1HiS;cZ6!>7cgxaA<IyE+UW>n82
z>Wz&ZD7<d38Q<z>`RK-*)fBhBb%dq&)1sqM!!eYuv4{4^VyrAtj>SD9teNm|q5W2}
zv5<834Jzll{f%Ijw(j0#>u)Ho+O5~CD)4yFg%ZSb!m9RhF+J)rp$r>-IQ56m)cD+W
zZ#gR$K0F(xPX4U2^6mF_yVSLdF%B?;UcZh53%|!<gb%3-8=nE829oCaA>=m$yfH=g
zwEbyb{_A^*d>HI?Hd@)BO(eGVwMOb`L=b&eU6v%%`;^zmTD-$<J-mwgK$PkFo@iol
zPc$(5XT?y=iQ5PEAKB5_-CVW#9Khz}LH8;2Aeg_e!TF}NQ84v`2#JI!-*(h_aE$yD
z95dN<ymR)fl-H?{#+$3%ZXnWa*Ps^gk4<oNX2y)Yj46Q~$^J7d7h{qsK!*w$D7w@I
z2IP!A7VRA}K$i*>1Pi^N4$;Bh^+%(5nF;kW&PxV0Sx4U}7|XDCBWi$BjJ!Ra$%a#W
zUVLMv%wcl4u8EQcD^MWs0tjL`3{=p^?T}1{i4K@g0m1#dU%*TaYWMDrK6eR<r$<Fz
z1A|rEMkcy%1YiLx0JG-D%TCf=?}dqdRbz{Me@gK(i}d~Fb&^&`5*dRm!#DH`3#Fyu
zCHWr|6TPFphdh~U!QhAqyAR>-aL6+Ilsj9=Np=TfLn1{e8QTxpFvY$^`WG$u0>E~4
zoNr%wlqLzZ%Gn6F4%toEsZkl1Et0}LQ-0LM1Es6pNXT=;w<G}(SU#2VHOXxYLl1>-
zbQ{k{YSu`y-r|Q)D_@<L&3I0Y2*kW%{cIXo^O<51pbwR^nQf|#90n_GW?{5Co2Lh9
zH`Cf`Q^g|tc$a%&Qd{1KLBrGm!bL$^n0WmnZdGU;Tepc}OWypAXbKlFIg_lVq+c_Y
zAwV4tBx2?<`wt&8FjsX08}{7MFj?jND;ddsQXhflXj|7~fr}@qlFWJ+iAJu<Gh{B*
z-29V<qIX@mroXBlHYZ<y=l09_L|9C|Kr(46J9~)XeLt$L3x6xiycLXo)%swf%vl^q
z&V=;NA$E#-kI2kEPfE;MBlc}nv4};VRtoVM>-N5tJ<Sv3aQ96H-BmV*njiVZ+L-u<
z-M#={r*qgaI$?I}QP<v@37Yq9$~_?>^z7t|q@Xo`9YNuui>3%0w)-ELEY$=FYEhCG
z?l7dPfBbWQyW9H+Mi)+aTGRb+o<I(vJ4ntvjU#vq!OuJx&N>?>kj>hlNX5#8O`qZ@
z5%v(938ki|mZ-)9gb~$LIo6zZo>r!`s`TMytY3F=sy&hppkX3dB9SE9zg7Ie9csQ#
zk{%2bU82$<wrt?uL(jFhGFxhsOA}F4v&L*9WW%acpO77;B~AiXhXbDlyTx;;-Mouq
zLY}w_dO87M$Ftu&>oabvjJ4VqMJJKP-LH?9CRJOX#l`_2Gz8cL<oB4T0=8I`p0N=r
z4(m3c6JAhlb0Y(R9jDKuv3zWtY<s<m==yz_z=<o7H1F_6S(b|_Lp_|{Jet~tXVNR#
z?Cv?-lCfCow2AmB;!qfROP$>&<;rc+E{hY;6JSSWLM=kWS&dzPUJnMoVmjT=xO?ox
z{4sii<Wn<mmU<TsWjSO@cYD3A-r?b{*m{^GL(jRh!!Q7VOkMDV@V<3n@AmhzRt+wf
z7cz|v`Y@qr-k#3i#^-qD_;e1)c6#4zdUh1lOsGkXhjO-9o4K_bT$yJiAJfnfJcg}^
z3XsC?B6wQE7y{QmTZ(b&JX!Fpdl-6_9BFhc$uERn&$PVF!-%#Vtl3wkin-hTwk}#J
zSCM!!9J*G4l>P7?VUyL%+fGx8EC_zmuTmus?_Ab;>X#n!IM-wb!%Ke@C*S)MobB)D
zg++gNHX_W4*-uD+WVg-J^iQ~T21$&5?1Hs;Z*1%jflmH~+^Fns^LbW}B0ClJrHR&I
z<XZw<+Jj$AaU_fDte;}oOz>WKExh`3*h2<FUXUygGfW3bjQ2)m67Xi#FkNC2`-~E1
z*fn>osL#iG#=7iduw~cmW6<j*%J=c1Y2#$1Rz4McG=XTJB(+$Z3qdY(?nf){_ZD~@
zP|2IlP*+(x*!E2d{FEeYa>6Ff_~cc}3%|)SFIV?@tY)f(5GfD(E(c4gR_*4dR{SWe
zMbl!hq2%inM;eC>twyn}A$V?GU4IwjPvqqyD;^J<)uH0)Pxm8eSC7tolgEq*4s2!o
zjrX*LFK!4JfA=>zCS%N%9oI1!`tj8f<c*70L3-UxsZ0mQ4iT8AJs&Tyc*Qfmq#s83
zSUpRTzJn2)MEY>PbJz}K8UM7y4H{4RX?3ErJx3|^u97~P%rr+cMG$!?*-7pYtK7vQ
zuK*7ZnnNp)Z~^gLbNY$XZ4*rok*0NSZdW*Jp$hRlzK5B$nQTe0%86#b4vdmU(kdJ#
zW|J2$jYN6AToZXz`CS;D?~fm%sOK0?-5)jfP@O6zH!34M9|ejJhH5Q+%u695chOdL
zdDSmTt%ub^dS%z(eii|_{MJBkI#?Pey3lHC)I+<Z@1T!~dR+l5od*GC0grCavv^%(
z3qff~N4bG?*H}GL^eZ&u_u3?+F54!ptTHz(ky3b`S%xZk=}Y}K?6`GEP5h@x%N_7{
zMcZVg&BafzA6t<9c)MhW66G;67}9Kc!H>;{GkW-v_`}8`@2dNQ(Wk2ZxL7`{S0ZD9
z;7#sS@VY|l_C!ZjJ*9I|OsOb3Kj3_KhAH~lV!8x3jrWTJu6b@G>n4-WJeT>ek<Q0Z
z!i(RZh1x4M(fLlnNNjDTNMDh6RW|9ls_pDtq4sOCWO!xR;n*)?-}HxGWJu`Vy(24!
zqzu&X6Nc9X1-Hive?+A25PWW;(?a(=XUzzwGov#9PN`+0@Rnad-BK1iHFotFKZwC3
znA-1fVDmX?ag8d9U;276Yi?oH2>zBNGifc}b7?agEG}xtW**I?_U0`Z1q4JT9(0!w
zXr2D3ZHhRT%)+8Bim<-vvteY)TT&w-9uucGWV($UFFH!u$d(R=)C>LkS$TCyHlWr0
zs(-#KyUEupBQZJc-H#ek;Ac~0Fe0ZvA)YuMfKpnh?jd>wW~hWr;BVW*sTkXzg}W~Z
zz-e}~8kK1JEKsr>+d1>UWbfBkwjeA67J16jm@;}vZk=t}$<Viw9B{Zv-uGJ)U8YC#
zS3yZv(|s4hlP!4xgez2spQ#S#lMtocUO$cJ8KXy;?mi@Z$Gm8EJt@1LwMD+9f8{Ri
zYOL9K{kCsiJnWM#mXJn_fO<hodS?z>UGps_e@CV^nHy4P<T^b{MmM;JBvZ;EohuAi
zNYJ{9!DM(w%9S^ItBb2QEoZ0@YX`5ORHZgrx`{sChK-*)X}!pv;QcGgGN@Us`Q5><
z)V}dOOjrsJ4K12+U<zEzj&U}Yz2Qw>cjdoQs7Yq_wCX%o*4^FnH|y_ym+Vwycj5X-
z$d!-p)agF9V;~)ngh4vkh|yEN$|$BgWC@j&lvb7WC=jAZI@TthT3@@;v(+@&R04)f
zmeLM0W!h0iN5VbHD@dIWYebjKoEM*(JacDI^K|0L()(JOMI?8HROlzx7*&jj-UKnM
z7g8tp4-h)vN>AH0QxWb{pmx-9er3hU)tkujH;X(MhP9EAq@LdSp%y%bXvsB(l$snB
zho3k=Q%N~5)ZXF-e4jcHxf&-KBQf}3>N*>Ot8c<weik#I<T#$VM~__IF}EM-g|+f_
z1m&mV@yCYJF(B&tdp)rm++-#aM>2M4Y~h7y7d#D{1G+%D)4N&UYqm=Z<~7}=LxUV@
z`;wzF<Y)V3lg>vnXh1{58a}g-p~{s)M>(pgAwij^;~eIdvd$Lty@DRaIsS~DGZO+U
zM1?hyc~Wq`KkkFDbr+qpu-0`k020W8dMoF&1k9j!DCIz*R<=RwIKL>&Ej$diI1249
zO-wUG((6dbrA%qbro2Ke@V4cva25&HosT@ifPE&&;hpz`%$UBrJ*{T=S9Tv_abiR#
zr8u)p@NpM96{MffXe~q~ogsANUy7sAM8c3K4j+=i{wuvpyrI1{k$LyV9v|}*O!9sr
z5^l>CA87xaz0YT^$szPO_*BFD0RW0Qjyf;#d<J-tG6Dh0tsY{f&i_QN?~QRbMK>Qb
zDq`p%sn1P(O-+sH!5O4)2Ksgl;&B$J5RGR;wunmc2jq^~S;hfmq%RoKJd@5AeY~~<
z%l7&iYdZ?-RQ<k1dro+!twk8_)BA|_r`(f}Y2P`t9UFRy4;nYV=&vw7s4a1v@_3_Z
z*F?M?Tv$Y=woGqoa`u~$`%x%)P3`Hr8jsQv<&$9{Eu*ofm6Ok3d!-$SZG8sI2+4!`
zQ|A}lhexfH(xYZO4r1Xg*v`3-sJBCZ!B8fFDnN!re-*dQ58X?;p>)>5+Gz=2y~<(O
z7B2{W=`H%!_VT_w<W{nX9(_NJV~W2+h!-sGS9N#zIM*0Ekgvzwr}(t=#ft)vb7nvO
z{iL%qVZ(^H&Dkg(pvNJk$hq_Zuiy!?14M`jZjxJvR{*Ik$fgyh<dC>jO--PJPQheS
zaeN$Bmh`Cb)*>wbvFH{M4Ti`46O~?{=V1`4cUvqp1sh;}z@kdSM61`Y&*Py|PXO}@
z?^Y~uDSA|352<bAUhH3eBH;a1x74=MzuDd`|2MtP+^l~Q;@qF#+5e%8Cs?!pu-fxy
zlNSa6CKSV*^N*oM$sG>R;(wFeu>ZFQDA@l%h5+_>5ftKSEWz*}kB)gJ7WxvSl*J;J
z{>9rFdAriZ(U*WV#s{nH;J>^%($zNJM(zyI=AWvA_vNaAi**MPOX`{U?*D%ADfxeL
z1et%7jR!Ne-<-BfbpeP$|N1aXDnaVz40Xf}K3~g7;9>g@-Ht2)<gPSWGGZA+1R(G1
zG5GQ9PZ05cEkz2dKsMvyzZfOl1%NCeEEgg)Kw62tLY)4WdyBJSpq3GwY!&S|n7cl%
z{O?d}kTczZT?1!yx6=g}?E@H;Fjlfe27(dxxr$`PQGc4BC(~?ihTr@==uz!&kmP%A
zq;mm!b24T#mUJLny>h9kI316joTvfmS-mzj^z(`#=#mj(l%8ApOi}2GYzSPrJ2dKJ
z>XuVTKU;(n6hKgj4(xW8s{Z+8=)?}lvXHZxU4PvIz<l9fd`84hFuj}zF;N2;qbpDX
zCt^$;L?(436Ma%opn$oli`#!c!WF2MM{YT1oji(!M}<3b0d+ST*lLLfRr~2oXQ76t
zxp@V-6(N8@snY0e)174inr<Z-N~QrGr{!MZxqO(B@QG8+`VTG{*zZTWAM1}N^Jw1w
zx+VCNtz_k#9|hJVlDgb0Ip`A~tG)za-?4Js!-FWoI_azUw9CwE8VyK;BK={*+P8=<
zfOzR+Xn|N?FE3@BkDXFi#d~@>msV1Y5o$}U72}Dr0`8U*7X&puTco?uI|3y+5^)Br
zt+dd3O>2BIa_tmLRIKm-q>Mv1v_p`@PSs);V9uE-*(&M&1Xxk3Ujm5LBR#5D7q`WJ
zK+AxQqgjl(ckVcJ+h+)S&{zVU-7O;p7S?bCSh%}YlHCvX2+j(<FMxBcOO8xcOUJ<O
zJG$#xp*!gs@-X`PV(QQU*DT5W1A7;6?FkR#hP}&H)`2qh(mo~p+E!Ky$MW!xV%|d<
zkmO}wg)YDZ6DE`7L8ub!*i(!lWbe)%z?wBj8<c&CeNLy#zBe{(PVJ>A-r8;fDlo%_
zvtH*<<R;K`dm><%eXRgb14o+wx!s5}ASrNWL<t#?-_qO(le#pcjR7nxb#Z?Hzf)B{
zfMu-!c*_nHDNdpQ_`%(?eddEP`q*WqR$TtUm3p#SQ0w%U+E@cW(Dh`-%PY!DUC1iy
zxsS=wr@OqX<{U|O8b3rE&`to?N`O8HzQM3%38mmvb56VX!76q0==RUV#I03M0N}HV
zQ)Xq)`&nUdwyqvv&;|fRSAceW1d8PRN$;HRpPmAqAJ4IA{N&12i|Lqi=9xK+!3;FJ
zXcFnc($y*=Fm&k{6Cw1Sv$*xaGRECgJ|ec6uL;n+-GmWcNd1OL6AsxhukGeTduVBa
zUXXdp)QN$Y-TT>{-xWJuOh=fdUc2F$ZGHgQWvZk7E@pS-P=MOvMY@?F_zi-v`EYQ%
zPh>H28N<Wyh3CH)nKd3y+IzShWQ>4AI*4TcP%XoVU^-WIc&JHiV(_cEPY#}200-m*
z#?)6=0xZ0--!fKUmuBXWIXFBvEfC5=&<gl`j4&6SbUyxC7-bEFi-HIq<GMS!0j3PP
zYIIkzCjda!yqP?8s+tcikOxW2^%ORoZ`2fRZd$}h==4cmY)yt9AEjpPV<MlUp{QK5
zCBmA2-ouO>`LbqjG}U_}XBj}=yCtQ79&zn`)<Spvs$I=5JwpgswX<SES*e}|M=SlS
z2t`m1&L2gobB*#w)c~l2$fRj9fW%5RCRtJE&D7maE6P2zcITFANf8*?off}>udsOq
znY6YlODrLPH9<Yvk35U%sL6u9W1_JIrb1ybtJ0rA6>ejI1{7*Ni1J;}g4r0+PPfqo
zTzc=_cM14wKT2hNe+#!UgxsVm+N@{sk&a%vw}*upU)YEZQM>weKelx7lP~Qnj~HiP
z@;gu3P3mO9tR*ps(c}7#GFn&kBtewsBp6LZI{Nsazv?$c?3oRx=s^$vW;$cnz>I{&
zc6#G}46k!cQxO)mBp|vhi_e{@)$78sqqE!rZdI7;-{u174LQeOvl=Lfg>1s0nkjud
ziYIAVek*yrl|%p(HAo09y_0OOIkh*|cwOEtTUaNkI130fXZT^vM}~%ge6x`>SLJuL
z2`Susx#qW1z51!RjVf=_K=ddtXeLU-L2aj8FX-d`$3BTs<D@L+L36b(3yEPbKf){X
z(qW{);uex;Rn(#9LeIv+KU)`e&}*PXOwDPSy$ej*pNlf&VU$9Z{zNfH90(->Npp#{
z#3f2-uF&+`mYQ3Pk#!c2{qac{TgbZO%YWU*{|Hp&^;}RP-+3bNXu(13goyQLv(Os&
zK9*hzf|>gLowq<>%};Ix$U3CHPUx0n4_`Nj6>!f5An|1B<X42xaA2d>&}T}A`02&G
z=+J(|tXr?3?snp}&r3;Q=Sc+2tnGqP!X9aU$2sT1ou!1Em|80He&-j*iis}3rezGc
zBvVWJW$;I$$eX@lqvlzY9bRuUN||T+n<#;pgA?l=`GoF!R_sITL^tzGb@;#-CLI+Y
z4SpG~Dx`llf%4aR4D1dDYE|zDod=cKeg<y?mO!hScXzF1h`79&P)_$-u{#Cm*6e2|
zPABe~y`7mhQ}m48`6KVO^m=Z>AvfnoE{+ehmxbr?ltZa1En7@40Mhp-*=isGS4SRT
z#O;Io)sIJm<Z9=*04y#>j(D)xDP9E_X2(Py%&MGxgOP6kHrtq*yY{RG6iNGwg5)tk
zaxtpjso8wvA7*sWb7h50V#7;@b&^HeGL(VmRwU32j)1X{n!cAITGGL(K%baoLH@CR
zW(7OpfbdCq4hoOec!v9YIXu!6A?wx;S?<wZ9$xmJdXx8a<5XEU`X>$E$JHxZ6P)lJ
zb%`Ub@={4#Vep@}cIhbNb{e;y+%)WuqH2&6G4bD;Pdq(hHnaA5Ai*n<6363KKzYgy
z{MUw#&KQYgaTg>c^>IFgWt8ZMt7h$+M}whP?2C%qZoNj|l8fCS*|HGx-o*C0*Z^SJ
z=bn((8aC42KO{sPx5`SH6D~05woTI7?daO^_N}Ce_0qaMZ_F2#9+r4NfD|~%Ii?|+
zo`OrJmOG}Fdrl;lkyi-J{i^%Q7N|<TER%@Qa>{=m>jCLyLy*4;vMJD7qq2XZgJY5{
z<*kfy5y!NQ7)NVEJVtAIAi|iYm{|8)NGhT{>I-fV4ZwL4u=J~905@b(C>BX@uX#I3
zGb@LuSw}^5%Y8@m7L|`)_EC4B|FDLlcF78l*`guvEc0|@#+4e6%L}y^WE@N%yO;=H
z6-i?Z(h2WM3n}D;%Ga1?I+$wdS<y_Y=>2h96q7kM{`|x>I<awjS=wBRntUSQ=4zAm
zji${eMoWsrXB23${-#<2JoYiw#PP^w7RM=qXNL+%$6#=oTc4;+JtFcRtz${sodMHl
zAojUl%43D9BlYZH>ZrZoeK-;KZmB<|P3AjcgcrHl{aw)Od=wE9ywUdSQbwnFbLDO4
zU9|YGYPw!ri=|%{zR3}Hr)HpwQ3-mCGQB?SHb>dS_dNRXY8?U3^HEVqtTXUjwQ*-(
zxV1AtF&1Z`Y46RS_DKy&ZW1da7#FvJA~;pLV9AT;c?pcT*JF`E3+wFAc<r5Ut6-RL
zfCPg^AA^h}&O`Px+l}O6k3td;#u9K+i+z26G&aJg(=V#0W-dyc+7LT3LG)J}(SuQ<
zMKx$bREA(HWQiMzMWJY)>yx0DNaOii2N$OBx!3DkU3II|;8I3>#j+QV%R7Eun&Y3u
z1;Q6(a#OzUUw<@$SGZ~oTO4#og<BKH*`Q;zSO`PA6&9C9<waT@HgSE$@S{h{6+cAF
z^bIp4JY3a2Clgu2D_~A0E;ZThvlhL^7Eyv5B6?Ci!g40G9sI_TxcineY2#k6geK$|
zd3^^8JNcJ6pFftP2JODDf<y@4LC7apg;4L8bQoqQ0E@w5ll7~w17imYPzE4{3=)cT
ze(uKo9M^St8pOneyVMJ&O>%sWD~?y8EHz9hp^tl(^uom&13o}vmTjUMhGj)d1Wj4%
z)%i8l7Cks|k5q`^@KNzwWB;YZFiN}P;=PAV@$d{;czCccyGksxiDYs-;VIx9{)y9L
zgA2kX+0SraN{~N23elZXV1m$uMpDuq-uq(boZs@G9$|CL&s{ex$7V-lea@N^vP~?$
zRg`31H?6hRi%(@Dgrm?1y_~3-xh@K<j>$iuXKWC1^>de^7g37qor}M>6dK+3%M8m*
zjy_iQs3=I1Q4!?GBm)l*$}8KeC_!NiS)F)=zRBP`d{UQp+QJg!s(3iHUo4b(>fxfz
z_*7wh2kb<;gRufaqvW9Hyrbv0R(^W1lKlMK;x2gi@Ov1l!wdTA2xYL?u=nuJ0KahH
zsbLTeh&Gx;?98~*ntrC`z{<$aOo|!uB=|ysBCJ>-ddKag_(Idf+h7kC_cLddQhjgX
z-8UMu<j*Ih%^B(|LB)7z2O<jMh5Cm=0<7qg2<(q7!TW;#q`O<MdVv(#D9IlApsH}3
zX3avC41&sV@;y`zn6_zas)vJB_#Y3_yv_Wb-elHJm9<MmHFw%3ym4zRsZ;#r?#kt>
zFgAG?yggc2h0W)wQ6stJ*{xn_izv;<46C+0X)|PP9hbIkgFt6TLl_W-*!%d+5#J(1
z!uRfrbWWwTUK_l0+!GQB#zhR>qUK}#k#f!fHJ{*k(^1s!@R%n;3KgMRryt`Pj4ub3
zB^jR&p-0u#oPSlvftA`GK*zTnxgL9ZGsz#0>QX;M%kI((!jWKU<irQ^C|p$AgQYLV
zh}Cg|@^#*8j3O#5xEKhIcvhzDje{zvV;^*sIlPZ{IQY(lZmv*zM2ir#YokYE^?xCP
zdqP_#D=2k<qSDh`d~AVr;rn48xBG`6g5>*mWF(vIl@Eu0P{yQ<I0u!R9q8=?@JRH#
zAFgU}pyDXQ(6kIQY&w#1f%_wNx^xA+eF`2|NA#+#heEt6hkr-e?V-L!7(LE4qZ*#L
zvmrQ%n2$;5qatb%sxA=-_VFfVz5^4&UBmU+Dpq9az|nV`Q%DNJlF9S85A<oda_FN}
z*@UBZeEf)9AL6H?;7sZw_t0Anf0`WPotR0|BcX54jJ^8lkV{`A-HV{5kjQX0^}1!`
zk2|mmq_QwELiyN=iRca<nu9r16#&*@B6z&wOemMNWnVn>*1dPwDogTQ2%Aja(ILEU
z7u4S5Ao~vgw&<GZNBWwTFneCLQCy-BR@i8Cd7UzH#sZ-b#tN(OPs0CWOmNJ@HjPbm
zB|Y5B642|hNv`+;%y18#FOeQnH?>dn{)CFAVvS}=bauc6j)4}3euswWm$#_P6?;L-
z22cm0$E*>g>e$>W;87MK%LogSVpiDQ)Y7vLkkUUY*dX;;U23}GS5KWwX$F?QI7u3w
z!i;EWV&UWTFbRhjLQKKE5vD={LKhpljRthWCG8kv0v)-xDBO|Bq2))@P#C@8sl<cw
zeW-7tT7V<fmW5^s!u<s@Tu;Nq&QfY)@#>@cw58Y5gw5<Tla29TlgE^g@zD9sL`++)
zlBSpI&t?Tpdfb2YtGmJdV5W}0%sUWedq$H_5QNF1^G7Ah#4)1F)<l6oVefT5c(2;^
zow+juO*3K79gw)2^w#=XMvMvZr=2?%W{ABq+@|pM&ANsy^Y`H)L7WrtRlQWaNY9CP
zUMT5Suf{*~{*fgee9a&?(^sARq%sypYw2_=)NxB7nQ!?F1#{>$*_5-k$L`@pBu5OV
zZN%(Eh75*X45vFRt=W{byo?>CLV;fgx}jBVcelEJoUgxSx|=FerEeI>%$O~Kuz}I!
zf)PU}Z-wX^9ITxvAq+xtRL1%I?(@G#b?Ph~W{Ulv*r(EByd(1}vv0k9Hx1`<mwT1f
zx$+w)J@NR725J7>O#o9f%~N$v9K!4B=2pB~Ul{V2ul8FJy2iSxCi#T{hF<>{7W`pY
zi}OVb{x0uKW&6T09OXu~kp*Qcy=+yaKb3;uiC_$5j3WPq&(@JTB*pzzu~R82mcXQp
z6?RNHf6Q-GM>wxv_6K%2p5Kli+65g3aWj@5QGengffuVLnc&gZU%vbt)j`jY2^t8U
z6)L8$mtkw=VtNWM^fs~(#?dg|Z~m$(14>Tm?PWT(ZCUws-GTf)*IGonR|9PYElrFA
zH0Y>;Vaj%Sx&BASb@m)<c8V&{rh|<7LZ0AYB&0))JcLe9R}A!rR)3vyaE$-)EIIJZ
z968Rs$dit$Lv1~9^T0@kNaL`(SLG8m`!o-rHw)iI<b=S3Ee{;RWBbSSN-Us5B=jdS
z;Qa^eF+VTlB)t!)l8EO;h1h6tNJwyTPQF*aOR2bE;%?5}Mc2~xDlgqeSO|j@<dRfr
zgO{a~dSa~C)JQC0;R3DT!C-nE2F10q9Lca<Xt5?8iJfH|MgN`ZA>HN3w2NRu=!@P)
z=zcfDr|9!dZboNSICKpdNk1rJb#HtFms-4)%3fXFJjxA%4z7uj`0|ZExhRH&yy7|5
zMEm?LQftklXS@Ci=ECEqL6dA10xMl#cDE>&Cd|STXVZgI*DS9H2A#koq)21j*RW|7
zhdeGzgCt+@teMwAe9JH2qlW{h0-dGRR7;f7j&ZAd{_PBUsVm&#uy58xwnOnOK9LRU
z85eq#C&l(?@%(X)bw@%4_yj^?#J(5(P(gG_zX5RXNMTr=AN5rdIEZlKTi8nm<NZ@T
zgC}+wGJiPY(3@qd`e8{XDruaaCRE{RUpERs_{rWR?6Y{t8Ec6a>`7Th0s<65o`Ssy
zA$$Zp(f+BsYR)i@^M!EENt3ehw26q4UjO2(-^i>1kFjA+slq(M_haHjjPS=O6)1kW
z^cXZhiia{QTh^aBK3XW9g2~bQ@+n>GXZg9z^^?8xoEj|Zy*N=C9PrFrw>73z5;#0(
zSWy@elnicV-~{iW*Mh`g7s=@~EmT`;4S3?hg{6%y__wVsgu{}~z~MZ*3kE&p%QQ96
zuZs=#m4sXAyE3@Y14Jd9IsxPsd6jvmvmelhJEPCccI2eIUt~ba?27rb7PHvOR|_Sp
z*&^LYs@JGve)#-8fA{sbSUL^k&xTWdp)n>%kH$fZ*P>+Iio>CP0NC$=On-{SR?u@p
z#qnm3lk5x90rxpdpR<}|><zC*+19Yqkm{btZF5lia-yz|?0r)U6Y~me^@qVVt<o>+
z-FXvf&S{^FJc|CXmMAtp0}Y$$W$=tn`)YAR5$0*&!!cr3&_4%NOhx=i8|;Z+F=<hA
zfPWNMQlv)f;r}EXwcl<vum`nri+Kn`CM~`IO+0s}(|`wx{hg!av36RrotW)@-0aJt
z_nx!X*T>Bx`C03vYh1f~GHck9UaL84YR=5M6rqaI%w3h6PRA+}9poO2butiWYI4ld
z)G|{*X1U{BYV?4$v&D*=;qLtVA2ZmF=<&Lh+GM!Rm#`R{gv6N25@nok1v_mndfyyZ
zkp=C&b3H>tb#BIUlPAVr@CL=}<AuRg)%2f`10QF*ugIMhnpW?@nQ7ObFk*pS#FMd^
zX{QzY06G)fXysazOF)?4qE963FsD|O#Loeh@Py7PU)(=Ieq1qMe6XyOX_o|aBVDB5
z2(It_KD#x0y#shCx5cJvp7$hIaG90vO!l{D7y9i{eC7g7*Sk#;RQ47?3=EeJYFd<7
zkf#3l-l3WKPY7BBS(#g(&f0)S&8Y5l*#ep^J6=gvitH&G-CIJF2#?_>lf*nL<czVe
zHNx;uL1q|8-i)O7AeZ3&%iExI=DyVTspI&e!^c1mpqvB!_NLIcy~&CpjMKaDzmT5s
z|I%UmFHzxtZM{7wfL#MUx0Q8LAHF$8{9AUIlS`vV-T2?6S?xw3pV)(rDobXIGaiph
zo;xqkug*TXxz@dw)5O@$fqQ@`0$--P3=?tk9t^VDG2VT0u<Nq=B;?GqHe>o?SSpur
z(o^|g;>IE8AtkzQf}J?w%(JnSqioDL<KfEOL4j*$)XB%jtD{-h-THrv9}OP>8)epY
zmR2w{IBxairsa=w&GxlxyDTa(4Czw}KHZ|ucxd9kOgD1+pHsreAOZE@lWDhQK&{w>
zv!SNv0%~sxm+1-{UQ3>yTnJu!X$7`mQ~&FBn9o^b=X4={<-4&qA1<$U0}9io*8hF6
zRya=dIe?7+s~<4m4gIfRAoqXNX4Z5&V1I|FA#gfgK^3^hA$1+VJeG5^kW_!dy2vp9
z3k7Kr+eGr@#mFCe7z&O(Mf~9^9?W)rrzeap&9`J4@Zfp`CPDw+PvjH&USjax^O@w(
zkEPk0>se`B0P~x}!ja&0n3uZ4SP|4LfjO7(<=dMPHjLq$<vTaL{@yM7UEQGDzN^LU
zpgbQVv9M67$R<o*Gl18lU%<uZ)0B~9S9c#jCgb|oIe3hg&Tn>gQ*Vy}WK4KdQ+syx
zBPkoej9$X_nsCJz{yE{_Dp?!>Av)6i-k1_)F&g7cc3BG!V8#4a;0FWFY2|elzOoMf
zop?U;W#DdA(J2&X|MsJED+3K1iVEy#jya%k)B+$k;XU`Q7GkOVw2^JMEHRG{X4<<Q
zAO2-~_frO%p1^!vu{N;QY$Jd<ELX20shO?3gZj?SzRcZRpDbe}C%`*yh%u$+n#c}@
zmS8%?b*Uq(w=1SbZ&Ac~D?TEK1kV8s;;;Ff>>;Kx-1Czp8>ZLquPD(6KIn;ltS)>4
zpg>~`t@6&O_-*#`9OJ#g2w0x3oB<=+x+CbBE7x$o!TLHhU{0-!$0a!{AG5p0D3J)|
z^9QhFds4vLt1zSKJ@pJ%aqsJP<y?ai!1az2*#YBfEgHaf<)ugVlvW-5G5iacQa1-u
ze;bhj;HQ4KgY4*72O>^;?-YZMG4wXK8{oBiD?r!fxnZ;_@yXZWPE)|fZ$AM5`VB0*
zMZiPK4m5y!T{*UU)(5@zQeWeQINKp+8%C+UyXOoiznYpZonh>(lz_pb8xV3#Fp!dO
zJv06jMgvQDbu2%SrLCj?{Uvz6)x%#Euv%IH&wf_}f?EUR+2zO1pY*NS0SB;ZJ%cRG
zV}GEFGO?32Io}uX?Q_pv%%1#h5>MD^({=*-Yo7qZk=qx~NxRJ`kZT7+*l@`T^F_`k
zFrwQpobM0*)5Y5@g)tP%lN*<T%>O!@AFbmrodL}YPcW)LH$Z80$F6~;Qu9~#sGRfn
z5#KcJ1e6!ZrpcR}?@y&00aa>K65UFsURHYicO-AuU^<F-eOJX3eD`F8yU!xwJb1)>
zh0bj28})!j^^}*#&V5+DQ%*niWTM;}s+f6;anZ(xkwZ2WV|Zl&Prg?3E}h+b`HcEp
zfRM&<o$c@;dSENDW|(XVJozdhQ*hJvhdon);AZG)%nJ=khxA>Ek>y(KzCWSkz_bFI
z+?%ZcSMAT>7{x;X!*c>~MF-$(7By&kzf^CfWk>&=pDAS+kNW1tFLXSxxGVdNrJ@di
zxuxyg1T05GiAG0_IiS5kPsi9}gv*BmXl}Oxa?p)8UKm6B@>TmYPl$8<(FWsF?k@q?
zXWanK92J~>JcU7@KL+NL=Ps>9Z!QluPxm;U)V2_byRY|u#B|X50<4@nE6v}#_of-2
z`y=q;fYR9*j5X~_HNUA|S$&-;32b-+gu$bl#66;^3zuw<QT?o!JWV**BF{60Pbw;u
zO4Wr!7S)Z9_euJ?E8M*tUuFI@RFBRyW61#cwFvhZFmX+k*B@%W)Tr-|bi^W!Wl1zD
zp@IH+Z`DR=lfMq)2bx5BAL$wmSk#1*0X{m-U5u0a94<2fSX**>Fd|tOdlrY(`9RE@
zSsu_;qOs=$--wo`w)1ooE}fBw7|?LhjgbQZZNvAw_hkX#BsD0ErwMu%VR;R_fTG~&
zHsd$Hb0c7l3hrWFaZn!!@5Q)ZalhI<OH<wRJD$jTHfLb}Q`_;8qeI1nb-BM-t~#Y7
zMiwn6oHbYnZ5Ngez8GmZ?_KflFrHH-i3nFbR*Jvu$dhk90_IvFNB!ZT72y2Nw*wk}
ztR!gD+y|A9%EX?Ku>rQWNW7i0AvH+VF~)AjQQE`jp+;eEv4CY!3|s%6xY2x2ns${R
ztq#W~`xZ`5Rf~DSL*6u%eZEFFshk_25O5eemTRlJ1=+R_g>*CmEBzBj63g2mITj!$
zk~r)?Pjw96-{DG;sjuB_@PPq>aBV>`i@M8raF$tN{qe?&ljH=<Qr=7%R=1cre3<gK
z`JTt@W`<;?a&2`H0Mh;E4dB%m<%NFR_~}e5BY=;6v?Bp<Kn$L+95(`rSE-obWKjXS
z%99P43sQjkZWYPfmYwdc+Jr5lvZVqT=|Su&9K~|WILPPs9}!4zWX?YWccIrx{aPB{
z;oDhYTw7)g$LYVxA$2?JICP(FkgyG8SXt?pM!AdQ2SlkP7ui`CmKeEhp8nX5rrF<e
z_lP+jbgFP@rq}%h=v<Bg-c(Rbr5gqtz(_6LWOC!b|F&TK6dA{uHZg_jQ=tFKOYp%0
z6RomLCKK&Cvow;_TJ9|%W9DS8q=$i?lBtQ99c|3KZ6QpcLH`onXAl9kFj-)_CH(Rw
zfTcL9xVj--68nf(mNm6A;*$QQ`!(jB4RkYUK&Dad=Vyp%lt_E2ch1-h{#d(JqQbHO
z>K2fga~A=Wlc#nIu<v2YoQ-6uMOa6Si*=!D@k%VBK;z3BYP<vB%@#Tyvj8yOv^d+N
zqVpKm(6*`I(Q!zJR8G699DQJnI`>3(vW?Y@2VEU$fW{)JC8jLIc-&=>Uceg_xQ!N5
z<*`(aEE61Q{lSwZMI^Y7LDPF2ds>D#<BycJ`e>?vHZS{e9GRD~>Q@?#)uw%7FFTz5
zJGsHlpUZ#Kkn6ad&WGvN)AINOyT+)u7^wyO%ioMkr4_xjVI?GPUGuM@uM7`I%#|xw
zl+)Tx(l}GjH>`Kn%G&*R#1F>bH#KjVyufVN&I{}8gYEOpq$HSkk8Dtl4aK*T0v&0~
zKnJo~rCJh+PxaA+zXSK7WFlMCJP&E88F!0uY$G6zR-o~HZar{HTexho+PXsfbgh?W
zMMK~Dz0#-Iq@Fuf_yU*?>}Mu_Q8r7<iA*nq^Mv8=c>KDBt0g{3z1{_b>&}y(Chm=z
zaRL#=^Ml#p+~taJ`H&K6iT!~hK?)<r_o4eeh-PnE)a6dKXWW}Moc$gKf<e^|dK8*b
zxUK^J<NbGRzD}%;dl{$se%PkX`>z;tS)YmmBBcGrxdNMTraKu)wZN_9Ea@fk>La|F
z99O_MUN9?nr76W2jU^oZBo+E$<|S*5^tTvl{An!sPjrEgnF|mq@|KOpkdh)Uxru&8
z2fK>KZj%JbVfEu3&5M>QieW(*JLoZ6U9CflzuNA(SkeAA5nzs7!7h|qh^ZSKQVowy
zDw<@LtkqcW`*i*z=6?|Po<UJ{Ya6Bt5?(SWL6QVPC5sXpkRYIfpa@77kR&un?j|Z3
z$&yo}sDMh4oI!G?$*BpFB{ZSQp<&kcJ?ERLnyH$qZ+^_5Q*ZUz)iyh=_1w>WUymL0
zUI4K)?BprsL!}y0oL%Hy*VoJLjOH`@ja>xmBFv4R02-sq%F@g)B||fkCBe<u^+J>X
zrP=QIO{JOL{-8d3yyA3EURIKZ!be0rE3Jp;%7FapW0T`&>A{WW2oev2U(kYjTr}2&
z$dD|O&<?m5RGiJr)W0xZ2d5~T+2x+b6ithh^0$TZW)m_k5iV>~$opVp3AdZxHG0~y
zm)B$1*Z0!ysx#-+;5vs_l(#Ozvd7D+B>72~`+*N|C_YGIoMF7#=R3tB&9SKfmkd(u
zZJh9!@#{I(MN=22ISy|5Ecnuu2gcN|=}3@@`DsBVS65Vv<edpP^5MAf>o^JNbPk?R
zqE3uNF^R-6ZT%)v9#<5ixkqDEE&O07KENfS-Dvv^hxo1$5-^zF@LQceh^DY*;=vro
z*wBWAFRFAfrH?$@YP95C8pl>mKhIB>?E6wA1SEK~m-t~S@t8g;q4EdqL<PJ9+*1*k
z>UKWtm~q+j_zXQ*b_?Hw_Cm?LSM~NwE?&Iynrk;8X~AAoObLew-b<CGyyN0dD>l9K
zmY4i#FCE3=u9l=Lowz8om`wRnES>Yha)afX+d930$88R;FSkPWBc(SE18w26mqQNA
zsP{jW2i|V3`I%G6kiz-AnIpS8@!=$KEAcma^B%4K@I^|;b3|3ckCzQ1^}z8{Sp>Sr
z<b9*ip7D%RVN8Mnk;ZkmNF2IOY{Ak>5~l8r<7P9q|HztUaPD97^b~y+P1ivsLakv-
zvwL_jE04;<qh5RZ&0LvYo(8=eKN#8Y@3?=SX4F1S$Bm=0x-?gX`7OTiqHDg?*pRQf
zwC8QM@Q7rvadC;N@^c2h<)q2ykZz~?3O^fY*-(<gU0DHEQ40RXE5=p}VQtTH{Ir<U
z(+p#TMu!=)#ZgiVLl^FK6c@olV8Lv$s#<bR3n?Cp_xcwpK69Z-K!ih~N(v#4%w35b
z>mz1zFWS+|z0#gdiF$6zAm29}sHS&+Qq^88Rx;nAu!VlFx`k)8-@zCzPV`}#XO2dC
zif!--SVY$eyD+&zaE%5iJGLCm>~A$iM`%vNYeUQ;GO??CjVqw(^UZpDQ)B(KW;y9A
z)H|di_I0XreVms3xYTUeFkE4UNWl%GG_EkGIS?z6gjlO&LdkcVG-Dq7b1Yr)!<?2E
z)DSMkMSsZJ@+x0*xlnY!)o=U?bJ)+AERwEmTi%-Ae>X+i>y>F&YGsgQ;94-X(U|pn
zy;fg`?@OK(xqEB{W+B=h8;X$PCcT0m=m%8pE1|}T6LD9Dj5XFDq@~EC9IsiVdY0E#
zJVd27A3_wUn$Y<{<2fNKtsRRN(;Hm(kZwlAG{x81wk>n$7^Bimn796*c=d&)F0EVM
zm*p~!XV90T{n#|1o*$wCZ-t4=pjC+-?nDX6lr;HANL0KF@Jmvo@mwdoYRcSseVxZR
z7G~R|B(8Un$$}t&;i;8%Y&{(=myJ6;7-lS{bRm8E3r7$oBasXx0ZFzO;D8Wd-+mG>
zer^01S-SF++NqVu<*s}U<4eCkwy`jOpUPx>QPXegKTq17Q^SL<d%*g4Ee*2u#ot6P
zAG(sF%x3R#G(O)YU2&(|;o%zmB8^**o^1)X^kLf&L-J7S!I~)9=vg$RyE70mjWuuH
zkVg59InG{b9i5!N@fNUHC{-wxMK+F`-IVZ4pB2sG#4}2Iq~XS?GppuY&QE=ZECl;o
zT<40DBm)ejj9NMkxQElBJxaz`*4Lx4;nq02+ExQZ#=Om!(sbvpSj&^(jPbqaW7DP@
z*#(*phzajhEa>^bkNPi9HcroV{P1pfpX1fRO79C*P0UE>Nm!|-!se-5TB5eNPm+bH
zrKqds7-mBTEd!+Hh<i5uZ-YyttkC>(qGH}k&oDcLf{jT;p89euhtE-Cv%;%rZh{bg
zB`Gt0)`mFyU!b2!{#8ck67DQnjVC&Bi!q(8y{amjp8F9q5a{RKJ=FQLMYs)QJ$@vP
z()T;PpO3%G5vWg1m#2lZbK3s?nwd!{#D9SihSK!l!b4d+?_9t8t|};byfLLSm(=35
z)1Q*KJh@{I?p9MJk7jpb<cJ(Gx#J(oOK=AyWO7|^c{p#wGR~O1#9?1|ah+&bWuC=P
zO{`1usd$sv#DZR0<r}7>#V|UA`+{CZ%_|#0CSqba$D^kv2q*lr&~3+S?!P}riT(1T
z(kO_P%yc+w=QCneI%&-@nr2bP^-!3vTr*nGv;Gvgn)!Ir*+s-5RD@p80`>Z_PcY3n
zKO_}cJdXhz15)Fc=ePxohn-QYXNm@0%+r&)wSAV^knI4sHyzIF@g!1K6jo%skqo`_
zf%2YYkpaA81`>tq1|-7rD9q-fs^SJ#@7T{bU|H|WI~l=gzG5p+PBlc1cT_h1?*(z2
zvJ^6*9=TwFh?YrUII*Ms)^-0RU4c7(?g@V*6Zx0k^1;&C;>Z+!-1WMN*sS@wR88FB
zhE%)IA)AeS4f1MC>P;2*FB!%YAs#DH?KWY8ovDm8T>NrT@YA|Ghxv77d2vt68=f(E
zHghPZR2dRyd<`_hzyo@Q-%XYm6jv7Bzk?rE`RG|Kh<_f;T$|_2%gBhdT;A|m5(t2^
zDo?&#b<Bjf@)>x|F-W8g@T&mnzK@G<bfPf}w^P!!d4I$%`&LMi&r$q8&Yy14v%&+3
zo9D%MLXA?bgK`)j*SF5;%B2Wx+SFI$T0iTddhK1R?vnZ%BvRwcUmR{m)<&p9L6!EK
z-tS;^o}p6fH`?F0bMWq~+Kk|rPyq*T;_l8k#EAR6f5I-)Pu2Ehp<JlXh(x(q<_QN!
zI7jEK<^BWjR$k6}nc${;fqnYP+#_19r?M={#~;1cWw8I=@k9~(yK_t}<X4>5UHpG>
z5{KNs8Fqo>+nWyq<8B^*26lh8E?Y`>5{}y^d}davw5$!Mx0xDd_SSM2E%{VkgRQFm
zK|H}hJ21rVu{+ZGYap{8Zs3wAXE!lb;pzv%_6=9V-U>7D&Np~Z#{P9&uRb*4Kyy8U
z8SOux8Pt*6Sw6Qy&~}85&Ab9@6#+rZsa;Cjl_%oZDo)mS8-g<Z;v%^q*k195bnjp5
zxXbO&Qb?ZTS(|#3TQo)EQ*`5J*FVFMMDO?Lbsry1572p2HNF3rKk$V-5(ueC?ha?O
zF941Kt+a7<+=~7%PhAz(soEE)JEw$9`)5yhPenVNGiQFyeUS4{fPUnF?km*}QA8n|
z4c(K8)5|$=vs}h^uQXy#P2cV_mI4QKztBtB5cMEK!@rJj<&s0kOXuO8LJH)hR!>KS
z46gTuRP?=T-U`gn+nc|LEnES37CQW;CI5`#!iuB?Y-k3gn>KF+oOi#c9)|?In`nXv
zr8c==kI3Z?hoj}0nt;sc<8NsA*E*i~QsY{R*8`(a8vp$lyF^Ej%y)HyPWG?7Nb_$0
zERONs%XRQ6bTsrK>-GOJZvU?|;s4WA{r|ve{O`Z<e*-xF@Bfd?nOLhwUZ6T!1Ozz&
zcArDjw#6VZMuijLw*HSI9YYM-48CAzya)uhzgmXmYR+d!cQS!shTjdPH-84~|EZnv
zInkb{c>najrvgumPk?Ft2DWY?P<t9D19#?M0jw36j*EaOlmI{mnFg2gdQq!b0f7O`
z=p~@HIZA>YWhWt>nkEX!EE8@TM;L7L0Sf2-etPwfKwj+be|}W+IYIBxW_$5g2$Xsb
z7n`uQNQIYu`==i0yxI+$-kY8lV7v2!&Dyns?Y(jxPoWzgIN@)CB)4)Py5FsM9XLdd
z3*1#YW0pZ8|D&4N_^aWh&JH05#+$1+y_EV1Cp<u-{Wv~ma}7F^bwALpY;R(NOu55y
zL_Eh!o*Ld{tY2=;Z2U{M3{{#7gS?8f24<l*I-#&-Jc-680Db5{LgpP%(<cH{<EK`6
z2%2aC#A6}wtL<)-Lvij(kUr;|{c>vHP^&FA<F%6_28n|}P_5$IH3vpFKIx4Lb3Guu
zbwWZ9z22*s{9f@rQXD_}Z?+y3hY1gPlE0h+r&sP?*Et9r*FXqL3&2Pe0%X5e!v{dT
z{B~^Z39%yp0mO&m!P9IpH1Of-i+kR5ZzSa%+9a}`4P?k)+%tuE3!tciCqwRRD3Xry
zIbIPQr~#7W&@s?-+;UvQXh!Lo?ZL^thn))(>P^D_=Zd`hI1^(<fg9rdZWd0{V3|@B
zvIdDjvOpEP1I&r>P*cDH_;B;TK!*2RZn_4gd{CcvLHG(CjeSUF<v}6lihxL~%_-Z(
zUp%B44%K(y5CMy&WGg}Nks3n-#q|=%lY2CfSoa)!igE&)a6FJbi~m{#ftSAk+kJOX
z?OyW&v;p+40dR!11IUPXjk%B&fIhxQP8b6r?|c3Nphe^x?`PoGt@fmicv7r0ao_}x
zyorHi%+en4Gx9#HgKR|QJ}Xk?djTlOQZ}n+@b*A6^nT3>tgx^1oFSM2_bCuwSiUr<
z{oB`IyxI?LOJzvaxQ2qbkJ@t+z5cs6F|KGxDerp3m-n3XR%V{P1saHf*S^2C<|=Wh
zJF*OP673dN#3p$}uSG*8ebzRQxbzcZl)(a1A%*0hkN%wi`l$;UP9N3KyiyQ)WsE2P
z22-d5?$dY^{HpFfDx5$%YxEJM0@0cuL;in@x}&8ZUnavos7m4ZuMm*`c`Q`>yaNXa
zltaViK$p*l1l}As2}nkf8puVVh;Tsmy&VO^sUpGS{<lp6Fi0N1WuO*Blwk>UfuBW}
zGhTdf${C@c%2mMc7b?t!KH04IO$^qzzT{9+L?w6n=4ijX-l&Nm5pD{c^50fg<?qti
zF3<-&Rg=T--m|P!M+L&6K*8|X?i(ybqCwfRnR#5F$saTYVQ?U2xr9qm?_Fjv{As*1
z@R@-FnPXH9fC{&fOE+{9-1TP0UpG;R=W_5i3W)(2o=1h|2BdM^z(O)S7V}(7U0aUp
z95iv9)NR=bC*J&u=KZ%<X7*g(VbgC<8b{uqT`>zzWVuhaS~G0tW*BnmqOxEMtpkW-
z3}g5*5SCx!RCJ_k3L5VqULy4fR<-F#sc`GO0NRIUFmidy1RmQTXLYtw?hGAPny#!s
z**7w_oMnL&)T)}gv;3cp|5)_g8_iWRF)TNs+*aU@Dv_T6D%I_$`(2!7RrBt7i@y8g
zR<>iLIH8hecT<kWdzlWZHT4s6AbDVWlQhc3!9VBhKhM=uQn>8Uk~01S%mheef3-o!
zS{zr4&oZP%78Evga(y&zX9i7=*ONaP{vguo_wpR#fm~NktHaIm@PXJ@t{mf{!3JO*
z5e^6clVl~*T+>)H0=R4|Afuw+kq7d=D`@{ypit(~+VQe9CuSc4sBPYYj;tv=NoC@N
zhjIP9-U-1z;OS9@Yt;fh?#Z4e@zF><6th_6w`=N(*_B5jCyA9&|G8&q&mYutqu;Xp
zBSW2kJ!t&#TjR9(+N)*M>B+YFm8(K<l9p&)ub*3_N?KLEMVvm2D+z*GdZ)r>6Gmf1
zjpMAyC8Q2p|JSgYv?ZqHmEnZLMG}mrG6u@8z;!Dw3l*B|zkKek8+0&v@m(Uz>|W-p
zlG$ph0INUD%v(-er+|@YuI>v#qd<2?Pf?}WZE)(Mzpi1r3xt09`<V7i9t$9e2<e{I
z0fBZ^V~FEJzLwzG`+G~<!w=KvXxwC0qbR|8hTR*zy%3gRM|YR4>K6dx=@+y(aEtFL
z+kM@ByYu=-S83l@J>59d`^UnoAOz9(AnmiU&^69qu(^EEeJHZfB+QwiNO;G0Io*9k
z)|zWKQJJ_(^|Nn)#~&+q@Bit`h6C=@J6aNsyXT}Nngfp=gISzZBZ3pyUX_uiM-%l>
zc3@8PH#R!O2j@0_t-dy<s-5t*9~n20ky)8BNI}c&-T3$Z=Ioa_mO*Y%#L+$W0$3hh
zi~mbI_2bJgJm_Cs>COL>AiFC(5c@ySGBfM{kFD(g)wbpMUUw{5u7zU$3<_%ymt{Th
zs9yh_sb>Xkq=MMMR~`#M8w{w?6bD|Xyx=$lkXpOIN;$k%1#y|<A@t4LBL6Vxb_e}K
zQ6nWO=wv7oL%TlOB^Byrpg<Sf;&^DiDd;<)&Mpjaz6iSKle$3G#;y`{7rMZ8mAZe_
z&;r4(z}=9K1Ki|Aj}+)@XrQm557`0voTot7tAaL4u61?zeGEajIm*<Ae)^LDWda1D
z_I@;IFAn|F{=wX6fz!2&^UY1^O}qoLT=8@be{t<0x#x8jo|lgvnIuAOC>7i(XxTpZ
zJArzbVHAbev8yA>)l>5Fxoy~rRa7581YP7}2EdrDGhfxM&qB3EWmQ|1MH@^SiF@dE
z)-}9un(u+_V#d@^B4ZgN3-0u2IU^B@4|z-8ut@3zIYyaM%&puOp}IpeXot}Skx}yj
zM%kDnss`GHkm$96ht=!_SBEpfU$;SGm+{RocD&2c_(P#Mi>h~Pk24Mxjykr{i56!&
z?kjX>I|miKU0gzSMqTA=b_yAG(d45~oYk*20SGt|`4DJ!N&MQI4F$yCkC-|tL8%*P
zcJJM%J)VGT5l=uV@o^orPZj|VS^#9kv~FCzb*o7q(A4O0v&P#BpHK4W5Kp&(HpeuV
zcr`D%HjjBA&(^<)=JL?Djs3k(6j(-L#t(Z&>?_g|-q<U(<5i*8$-oY^IS1hVD6ZEV
zoXw6G4f&F6J7w1l3HbQn9$S=(8VgpNoOs}?*M0Nx3uD=xwdguM*+X5l)LK+I7Mz#4
zV<*kQH{>ge<xr2<4z#abFMJWjePpcQ&1qYL3<IN#v!9+We*~%XQHacuBTxnM`e66*
z4KndB0vL?;I9QasK+jue_5@;ugU)NVzQ&`@pK)i8Q07x}>N>jFnL?f9_SL_q%XI8(
zwp)i~3s(p$Kc_PAyJO(x0EHFhqI8_%3RcF!(rJzxhqV_AeStQ<+JFaL$sgvQ*!`>O
zTi6PL`Wp|H+4|4-N^E|-dnenv`q7{Ay9R}a^z%Q15U=V9_y(GyL~dH|iw{xE_ffu{
z@|z3D<XGd|iE^jtDtVSSgQ*rPbSC_Acq(L_!}W#a13E`GV88#qJg9udhW<04SLZ{!
zVApr0dU|~{(nJnIuj~fPG`uzi)eY#Yp&fO+ni#{-2ClrmykKucM`JJLrk46WZKJf6
zL9I&phF|T)bsI%pg7J4jI-)-Rev0pQm`NP4YZt+G`xJ)-+jqgfzhCg|a-m}f#E~8~
z5eC6Th6z2Q+abSznmcyy%%4x}6qpdaLRM9o>h9c$UY74idx(Eo7?;`uF3$uIRVWDZ
zCn?7@|B2IjvNG`&Q2}@+4kKkH<11B-g3$7i4Biv<TXcR{*B`tkJ1)ABkuLZR_j&`%
zbk~tPC+o$DfPPk<e-XP^QjJqjU%#lj17!v~@?<`Xr!ST(r-9}x!8$wqW>*3WAG8ma
z1V4SvE8?!LU9rz62Z+w5=X11JO?e5skC~~i9?AE70}7XvBacTK+(AU$dEDr8cF=)%
zoW5l?el8bqBxkFF-1C(aWKKVSw5Q~*fD&+nQh!XK4(SB%D=qQv7mWPVP<vkz6bWmM
z3n>}NDX%^}>;&(HnMDf{hmyPgL%T0?&~FjVw`<4CBIbaFKoA5{Eg}@-(0<pj)^c?!
zM(PgXQuBdaquz<<peidMvoE&OYEumh{Ywtxih>o{Hx9(F42ce~f=H3rv()>s5<7Qy
z6f!`}W6!EV!D|Kz=S1|y{_V5+k@DPx*fgCW>}0|Nx|u})cbEuH?#J{}6+OUZ>LM5I
z*2FM>s8|C5&nz|YIJ-WH0qc4e#4FD;yam}ssK%C;!UB63C>pGgU-VQ?OB47(PtO9{
z-yL`(vOvuD^wG_{8zU`G#CY7G9d#!Z4m@E<&A3P56)830P2z1ds^h+BxIC`B4!T#%
zDZ0skaa|e?y84TN6=#ms$eo4#!4ckyJbOIq>naYXSy|P5a%7zt*$HJLhqa~#ul+!c
zZQ*Z+x#CY|&V7tN_l%CY=Y|!k9NI_e14GD~M6*7dt@n_6`I1mgK;5>{RNW8h{dZ3W
z%>{ox+?8(qZc{#%Vdn6NJBX+I2XtNGxSH;VLORXM>StT8k@$>^xF|V%&AN9Jv&A>M
zbCZc*?V^e@pyd@$o`IMP9d7ah+ti`UX~fR&F4NyM0ZAM_$@zC>9Q;M?oO>w|reEWA
zzK8h*z63|{gZ`^y3&cn-U=GLtS?uA|_eoqH^1Y=qJZLd7|IluAkVPKG`~-n+-Bbf^
z27T*KP~9^eXgCe}uAe-Z_;Vn*m7ZN;OQ#sdPD`evbBCv`5zb)*l{!8k!G|=eDYa?b
zs<SKV;IZhqk(j&a=hez7lk+D6ma0<IEuV%=lsPNyC<~kh-F@2|r7|5*Y!2HcYft^k
z3IcI?FU;D%rwH`n5M!PPCKbB)E}{CGn4R-b*^=6JyF5dZBJ4F0HZjThd8tjQ%`~@E
zi)TS~jJ)%|HsBT0Y;Dot3|>wT$d(*y%`!KF5>0EhFO3a&-9X0nQRJl6hSkB0c-?ly
zBh?;X+wTlatgixV^iFaMZSzWXjGR3%X@7o`M&BdtD5eBG_X$^{OH~yG5vjayf8-WD
zD!t4oWSdLyqesj0sl7jiSPp#6zZA=7a`dPY)D7R&=e!wGZ$?QnA7LHVglkzl14nEW
zXAmvmg^z(?l8I0cz7-I2eJkeOo4%3-I)5xLlX%gWERzQo5CF*o=s4|v9Y8bcw5%sT
zj%8v>64rQ$+n%bhYVR$s7y~224|~cr(08}QCf<YIHMbrMls@-<pFJ}ouc7|63G6Bi
z4z75AiQ;vuS=u!xp7jJn5)AS%J}>e34h^4YZ3;(Vi&41$BiTOcbFt$T*gLMuzRu7C
z2y#BFz64~W;C}Y4tfb7<k9GTpytInInMG03vD5F1tY!h-`FIJT^NdaRx~<Ks9K+M!
zp7#v7U<2SaL{CPB4HxZwabhw1V1d88&{3a+KYDK7kdE`^>`&Dfs_7*H&*@*JexRlf
zL){U(6qefNA99shtX2Y!v-_>{!ZdJIrf;7{h2?1-h>wmW`o(?j0^1A|2Cgd48k>vC
z)Uco{9tsTS6mW1&ZUbMt^4`MhHli=A%tV>i7mK4GrCKEDtttn9Nl_EKPv#TdBmcl*
zK~I86V~5_r?&&8g-gK8|&?pCt-j6=*b#9CcFQvplsu?O$v{%B_l&TV>&sT|>gT&Zo
z+3a(`ga0`kph#U~&8ZqKA<W(sqMJg4DMZpQ_k_xkL+mcIl;efs!m;M?DX_~hs@0>E
z@40hJ(PWZ~l5e9+48hoQvOy{Q5Oq#wGiQ-U!E#CLdZhn(*R=OrJ)Jn)8C?;T`<dKN
zX{hDpA6&k2g`2p8g!AJ^#tZUSL~q|T`=$28??X&@4&l+XVh}@zq%$SU9To46p|*mO
zV}AN6nd;zVQ;&*_wPTaogMZ?6u-~gud%43tkA1WRyxzVLHC6}9vun-uH%KjAU^kI!
zwGpqk^3zuh=%d66rKh`1d>g6FaR1DQ!Y_IA)P;)KywR`7>P~015EkRBcH9J&862)V
z*Nr?56lqR>V4hj{Y)X<Oyldct*s`B@t8+%gJMc7m?J1q$4Hyr->8e}x!sZ|m;XT}O
zs`=+>{9>SxmnrK(&t<4AzB&$ex^8(?j__{IWA!oo+Q+jrwu#<~_HXxV>i#?s`hH|<
zU_pm6O?##>AMSA{EPW*_HNTf2ps=u?B{Of=cQo|vu+>of3%3X(jj#*OsYT#^*K;Kw
ztlOwtby$6LrCQ4tEWszkrBhE`yd|8z*z&vNN*%G<h|hmtOY)_@!oxs=W=%mD+Z3dB
zc@R$-Zy>e`abxou3D~If&7uRzgHieUPHA?*HMvraiq?;FdWz5+{oY6fjT@oXPWx-6
z@|~7!rWeCg|4b6jy|8R~f}m{^u$%v3f`83=%Bt~P+;Q|^VZWvL6yZW1J#gn2wI^Xp
z34$Y1VlISZ`xL?_cu>Uk+_$B+GiLHHa)#1r6ESLRM9w@WsH?H>U4|VYCGjL_ivYv3
z&(Yf7T-<=_lLK3|wF?fK@DW(B(cm{^*<>9{ubPsQ)(k>M%}0&DBuP1zwjd%fU`^30
zX-h4WM|JuR_BBllS8wbq#|Ia}wuej%ZRN@==^yfNTRMprR?2pEH9biSn`L`=+Qt)M
z<BWflQ6yoz6?cpeA3CE!pNENYM)Pyy1)l(yi<B(wd%`H6-1BMz73(AQWG}{;W}s@>
zrWI%WwwvcJmT)c4m!ouBd2hAn95&<7@#2-83$f-k0sh9u`aoqS;y7I?{EiOrnBD#d
z^XkzST}-Q_AErXcC0zV@ZK%R5Yv*V0P_0Nhh2)<%DhxcY@$!>*$21qWR9q(FdJ$1f
znD1uZ+35H&?|Age1ZL`SmhYpCY{PS&NAc>$OXP1pk~o-8jCo~l-Y&-hG|0eS`7cE1
zl#%t^5qS0FjAFc+kP4@-3r4cbfRZP*DWQ|zEfNVo!PkY@_Ogau-3cjk54WA6H$JX%
z<Ekp{Xzu3YAXjL14$fa5_kUM?-d1mH?zVgm(>Yt7ZR?t0N7mSRuBU74NNK3Tn8|Y|
zDIPVllCTnl4U3ps^>M_7noKkLf2m@Qrl2=lNV+)8-SA_j><Z#qh(19oRw;hx#Ntnw
zmBj?)-h_*?Y53oq9l46`90Aw3F4SsHBkdz;ON)5YEvqmtF+_Ecz*&#Uk+cqz`hECM
zZKCCC7up75u6+)*+77Q-CejR}B|z>_LvCK6?YMnotxYz_ZIo6%jtb-owK86YY<NCp
zNh04;G}YPzgf(K=Bb<^XkV!8LxJ!whLoWEu=6sI)4sR8mR~ZZERj)Tyhm|K;asJAi
z8H*DcZQCIgKuw#Fsf{{vhar4CQhdr=70ImU1{^*_&)1HY(SM2LD5{=I7QRZKwVC=>
zzgskBT;ZR|;raBkdg-XcX~I%B5hCiAR^j@m#%A*|g^>uZq4fFk)$$(EHNz7lhgS0+
zY*?7ZI52FjAvM`XQGKF<v>7$kAcU>vec1nE?YP|OwRxMfl<JSb(ocM-P+RjnpU-j=
z4w%2&zejb@lp>?FgJnYO>&V3;@q~7?*gQvYgrnYg00nKg!UGZ=IHwHH^FRp)>@B%z
zYxoy<pgzf0n>jsc8Z@qNV+n*BMWV&^raN!DwD@3o;U`c*RMlgqlzOZB&aI52_Ob6h
zyRM5a<z?yYjSJW3Diw8Mo|jooy?T4j*B(ynte*tUr4e666VcLLGUd(+zu;_uhAU4~
z_;*>)jf)Y`Z3-9I?`+W;@X#BR`OBe+Ofl>?4VY38JkOC2NF5bM7Mkc@XJE_)zoVOf
z?kx1IjWeKmo->$?$7>yTzRILS!+(iduc|41k)lBpb?~5&g%nA#im<SHXX)r$WDY@r
zVGSAX%~tS<Zh@KoE2nS7J|O7N%ralXw0Z8D>3iz^wvK{^I*lX-*G=sY<*O5>{z{)O
z<K0#xq(-X?3)6L~&L#h(=VDKFS`t^ePxBgkHTtt&^Uh#D8{w^Yk8VmQT;O>b5w?+e
z6u~%iJg0Z~!NawsSR@<VdvyUuy08<q5S&YD4GXWYyB&!^`B`8gX|a<4K0#)A7B-)R
z4870gfsYJ)P+{*=y&+t>6HctaJ7nUclygV*vm{rzKmF9IotpT}IMp7_`^FP{6Hfsn
z6Bx5r1Vc?vs|D87)c%x%rL;iWptY*(z5}CZ{1nTp^_LU9hdt6yU}SQG`LM}O0#`9z
zYiSX*z{1zk?8{-*mt>jgpe<EOc@Z+Gm8Bg6*l)ZeULjK>x`qWy$b@qI#7AsmS*3zW
zTtQ1h<=3fnO)=N`UmPe5YmHAmndvi{?a5E<JNzTauFo4c%cst*C9xB}&*f{Cc@51K
z<|;#9vYDYYEw7s@(F(m|uT;K*9=?iJx<zBlDSXrRO>B5aptTsuwpqp0q5?T5DsM#n
zioH#!7Ed!){MX6GEn9-ZtF;gB*AY1^^0K8O#17qCit(L-?8b}=eXq>sv@B7K{yT5x
zkWNUHtBxt-M5tu%T+6FQgFkLr!esuX7g6E?H}7pxSv9r5n14=9T;Xg)Tt^y{5Xo`E
zhbVf$o42r|246F$@G>seDyZD>B;?5XM==u`Nw)&%S_o8jO=auG2sv(|F=RP=-hEm8
z5jm}T>jlQg;bE|NoM@0l)>Po^Hunn*!JB5fnDMJ<*hg=4vD?-p!Nt)f#uaMMPHVfn
zf)d8<elwZTxndnhU9q-mJPE@ZNPJd=XzvbYpXHG!S)dyOL-$diS!L~JvWjlbyR&%&
z8S-Sx=q}r};D`)*3h(X7jdhxKGgQQ5Snos!<I8m&6MV#B`CDGW6KsK3bKYqSKXY~f
zXC>Eb!+oP9rWl9b=yI_exyf$M{82wWNOnX~rNXp3w87C%OfxXUKT^lJtlc!Qj#j}i
zdSo(toVw4(@4bU+<4&v;&A>#%X(y?YS9N5oOQzhMq?$b^N_xom!+@uD`q)y0&q0Po
zI)MYbjzkm>isGlvI1!93ioWw>|J|yKnEN(;%RTqD?ryKWV0*oMt)w*gV)4{9?tF3{
zBD>`inEqH$Fdw<Zgw7d}xPNj(hUSts2O-I;X>twC9UokKNa^BuNr6l*c(Dgm7h%v^
z9OOz&>d%QBTy>TGqQomgtqNU<R9Wr$S-**-AV!R9=<LbLFOxaA98j>>(*Z8XX&qe?
zSMtO6qZ6*8raO_=8U#K&QXW5pllJ?<k5d8!c)qMvi^0LGS1j~+P2ZM5B)qe6m)Ox~
zwM}teA5rn4lBh;V9TrwNGJV{pIwvQF3|D1pMnxdp)TtzXzrFTo7;k^Em*h&Avsu1F
z>@!&TO#k&PL9nXeCpM=UZ<2pTS_+gz!1t41N&G{?OXdUrBfjLW)Ld2$r>=st42dbd
z+__ecRPv#hR7!P>vc7At*Ze)PumN!cPIY|ED)e_kkU7j@PJB)LyH98L_D8Q?1HeZA
zfw-|BH|0~Wz36XF;_Ww-PPTaUAeUl2v7UI>P{nWTKy18G%vjqmMPfIVcR@hHj#p>6
zt#D#M!LX|AV5Y2p>H4&ddBXOC=?SIuBrDyx@jXqIxn%0m*5=}1TsKAU;w}1Me)th{
zaW~!S%QP1mL($Ru<K2oZj2l^wk4Pxu+&wWtcotT8eKl;qn*Nw*vu-WNYt`ZD+_i1k
zk7DFaaDzMnHwZNv!|tu6XBx$PzG!hK{$tbK_G<qWe9D_|1GtMY>IVtIXWMG%;?{fZ
z?lV3YDXi{0OLsfMD2RIOYgx^yg*By2J{!cRTF)vlS+lHpO?S;s<WWdOiby;h6)V5`
z)!gasMZO<coQSG^3r0}G!W9JzX|VUMve-=MzyIV#*#rL_eYWTX=M%W-O>jY$$}%pd
zln2}wv1Wdi`nC*h#aUYW<TdyF-aUE7B=s7eOXY&4TB5P<4Gwwk(Pcht`Q@N_U_NVV
zJN;_3hp@SN;RRjkeCmBpTCegP5sqqrLv9*UVKJja^V~MW5PnxT)5?%x-nNq#;Z1wD
zqH#<p-?)3V;0E&sJ6`Rd?OZQdwNDp)yZ=iVbx^<e=;j+nlL^J{>p^halSMfBUF9Qt
zFA>W!aaGkBAGUU}d}?d@3Vh8eQG_~uskM8cepM^y+z*-Sqw0!PI8k~&EsX4^om1ub
zL|!9isS!zdnwZ$_f*pLMf#l*><EgcgGA5yjsQ`(npGRL8XI2M0ExoR7z?rAKcBv>S
z;p6GVaXm-J@pmUA_s6Ge+LuUA;0}KGr;p$Iz*!QaoF<m98LaD(-^Bpm^7rV(<u@l0
zrD>|T;G)~b9@56rT_$pX46!;Y9`G>qjDGvn;p`~x5HB}6X(R^weOz>GJQ{y*pDlej
z!>qJ6U!XeO!?W`I;-ar*in`{0`RJ(>#n|45T{PB{d?IjF?Cj0*!Qg7v;D9{=^4#or
zy0#rV=_A#&D5K9*Jrn%cmS0U3CTe_*gKCbsg>C11iEocUccXxA<9L8vO6j3gY*;;4
zZM=Unxq1Lkv1LGMN9#i0l%>3>0#EcQb@g6Mc4&5O#1|wzCaP@BVuWpAV5#sM8a%s}
zz0Q+wW=LP*O)F$i?OH^4#WSN0naO<kK1V$K6Gb{znAP8!vmrSJ$K#v6_D_w@g`LDE
zI#2A+4p^I8)<2<>D!{7vnz;LlV`FedJ%!Dp=pP*w$cB?W8yo5D`dX28Sf3EHzEQtS
zeX5(U$zu#cnZ)@{2-4j>(jCJpO-Xq|y0YODNoah$6k@ZuQEr2oF0dkrz2+@2SJ@<Y
zzAd*Xx9Rh7a?9i(k$ICbo%sxZ&_Eo_&})jj6N0b|5S-=r>5avoiM)+wbe!Nhd|~H=
zDZYxPvyD}NH6hmBUtP5FW29)?x*)4-epnLe<~sW1=CrCibGBvp_;GOX-ytQ}?lg5x
z25blH2Mgb$-iDE%AD+}J?aJX<-_}#bDkq(i-$tcwzpj{4$^BDPI2Kp5Er>EzV<W@w
zG<{J#DGTm8IcX*66c|!CAKE;bo+HIl%~mS2M<6KYKNP-Sez9IzWFrp?rdBuv)uHL6
z!WEbK`{sq;hL!2i8$s0Q*v~?b>%UH+#RI4tn`yRLT5lW&^Y&nTf8(|cNwm1w7IuE-
z1nUNS7KhiZs}oMMy{w-<u?fFwqgq}&!ar|A!l5R`iE64)I^WU^?fs+u&^U~tap=8c
z-udlL!5XyE#+UW30o+IJnqH0oVVh9!qv*W@vU?3KC?utDM5-_kuG^9|o>kI4_7uli
z9|ch>FoI32>EvL53W*jUURIhGzk?FM?<$gTz&yhXXwV!1)CwdV<oBDMN&fyP0qr&m
z_yGL)w@T0-3I0>~`u`lJ`u}p+JjuDb(BE%VV>_O(mEc6RkxtBicr17e^c9UOfBTXB
z``z4rbyYw!AsWb_LC+In9f7)@0QB%|7=SVU{TZgTj=<p^4~@aNj%V72JpnuAnHT)@
zVJ&8bV_!r$m}$_1jOwrx7U%=;d8*6)F9Iqhn1-zqtnKe7Vd`vQ2ozB$BJw{+!05MO
zG$ow}<v$hx8|}FqpO$#V<h~C<cwBGsjXBA`U;Sd;7aPipTHjJag8tb_Fu5HK+FZ%_
zS~uULs;ju8_k8&`Azw<~Iz%Ic8W~O?j}!+)%gWQe0$%}WQ#shY_hv=yA0BQLR3&&g
zW|iP)seEq|v%2&R*0F;DL8ViF(-#j~7w*7o0jh5VeH?*?oD7>ex6Jbo0L4$o2Snx)
zLDllTzS>w0ivxPWcW4+6ergy3KSFW2I4A(uWNhVndW7~h#vm#@qz)j4qQl?6)-I*E
z{v6M~n7bo%+sb6iH@ITUA(2vAj?xVJa}*&|-J)p5ks9Qk$SCs$L?b@-<G$e6O(^72
z@}g^jr;*;vcb7Xe!vyuGk4MTAfb3m+3bjfo6mI(LerC^GfdQ@$x_x<E80Zvb<+XXI
z3%qhG4*C0FnrtFupDES4tCa}Uq7}WofiYmdG4|Q=XD9deMSD(b`G9FLI`70m{B|^3
zmp73W%6|dfJkRy|Z-^-~LMeHx=N`UOx9%AO0oOS2HW^HV^k4#%UxPwZixdtB#Un6S
zRPW>h7({#{;YH@(WEjLEgp4VW<twOH_YCR-l?n-fI)4$&#<7AF=T*+!afm^AGEEHn
z2KvhoFmgZCGQUvvCzu364*z>I+3A-OpD%ANmYh4gN#f;tndXMhH*=i^^UC*N1OCV?
zE>s&yObw806Eu8qv1n>El*-a`>bPOsgp)!8Lr*CUNhUNt=AS-j2t(<lAo)|S^x0-n
zjuCZV=$%hzpi~nn|Fev8pbj$DECN7jzAI-5G-W0Jub1Thq9wd^DGlnPCAi)leEJ1E
z8p|V_pt)d$f3nL8JROi_%2yxxRcI<kC$LlU!{PAkvBD`dr8dtWhpVEEnt0LXa{6ZE
z%)jqVmeDDqYOfCTpSz&WMI=Bn(O}F$a<d?er0vl>xMD&Dh!(z8sz90nNb9YA{3#b$
zir{~tm;gp-Z~(@XE-q>pcxX$`g>b#DuiUIV=>+XFgApV5sVZn-ssM20dH*F&*6-NW
zFWoEn=FXfl1saapUR29_|K5`NqI1bFW#@C9gC)R874VfgsmVruW<GbE4qP1gk-xz!
z`U>2f0Z9fnU*kKoW%s^Wg)BqXG7mEF_yZ%y2+^A~eCO*_1DqX%-CEggDQ=YlCX_C~
zTGVR6yjT<G*jB57LvAl$HWRi!XgHwp_36&UnuQxc+_k&~(Xq!t1H5YuK+$l(lvUy(
z63F}$@KftN?&|^x85;hgy;*f$)kGr2K@5$r)I>qT?_Qla%~;(UOCxdB+!564oYScy
z6DZNV`niW>Rl{qJPFoYl_Jf$CGwvL+_aFyTr=O!;(*wkR<|9VVa7#CqX0?UpiUc_0
zd;!$+`<$`tL<7~=<VdVkmoIqn2?DR$i@ZcPZ_tD^Y$<p^Ls#_KM*xm3n+;;7{JKco
z@;J9%7_|WWT}bQt2Z{S~pgFkaJ}cMkRu12j9*t)8Ih-xsnD{mhwNnbTM8k+t4)}lH
zlr~dU&`IA(+d1z+?s>Aeoy~M9*M+fc!<2b+U^`C#XduSBjCC&G6uI{S;kFKPX}Z%O
zP&GQXQh1jIyukt=Z^3B6{1y)z3cBByb$LX}2=9rkn9*&{fVaZ;t%W^1zb<#SUSKjP
zfHIX9_`Ojf2V|GQck97VA*$1BmNQ4a?kJw$RWqcrb9%Mud`89|vs7oaiK|VrXWLPu
z=Ao)^wNu<u5j2-r+Y5Z;n``03a6g1%?Y)&LwUE~=S=X}cL1mTo7;snoxV|UUlUzmJ
zBK0%+>QUvB-mAs=C<Ox&12$+>HhwsE!>dtmEMTp5Xd5tc52A_jP~>?N(I(vqP=$*)
zD0LZhloM-9?q_tAy8UF0_}?i+tc&zT99c(@Doa5@d(sU!mWjA`R9?Sr%2NSJv;YT8
zVzmkB(Dlj1^Zi{Xmi<eAQm`!&XAG#GZ+;5q{iV1#o4sV<j=``V<UfE##qgj=%jytI
zv$p?gi{+z<Ak5?O7|+Fiq?+v5%G3o61bUah&8yI&SLe%Q3Raqj;0klezn+g+!eR(H
z9Ur)CchVmB?1PcaxP13o+dtCIrvSik^l1}TKu^XNJ=7mS<idVofoV_krRZFN_i|-C
z%Iy&9AGdgoBAP<Au{`{9(4d08I%x9uKZc8LGD8<@`UAN8+lLB6t<0uMsh<pnmruAE
zFVou(u7lxSIc(N#YwfbW;57tV;ZBdI3Q$V<k+Q&02g~z1d*DGgunq=?<#-3Q^Yn$f
zp)8kA9BmXYRom7a*PN<h^v7yI=r-4tPE>%rjq@!WkY!ythVEgzn5SyanqlwQ|E}CS
z4Ix${z2ehOww23uCtZ3Br!!8yfeaedW+zLS^WqG{=A9QHKnakO$do^&=}C4X=;j?;
z(Av62jP0Rpqz)NbGUcvNrbWLAS6*{q4<H49N!VyHlLr50)((m0^yEe=BkaWz%cTM)
zj?T5od#($iSmyrL)I2R=zYj^aGpPt352SY~e6(&WP$&+f3wqQA3LAoOr3Pju{AIZ&
zM?j#as#c|*F033q_46_r5oYh!TqQHTDUG7t&VVQfFQM#aZSZ+qhK<8s$y>(WU79dz
z4vq;Wb}0L~s`w7pb^LB8Fz;@$x_4YeN0^oIxl<kc7G%3<5;l}>JO*ph%peR6nE}3E
zNKBX^o1OOlCH(iEvH=XJ^PM<qFj?r_r1H-|SJ$`NVB+BUlI%B>LM}64D@AgVdnrMJ
zOXoRx0IB?g1rd(RN{>8;$AA-3zcw7u5q32hiMRPJ6H8rEUh;4+4x@dkiD=OFN8(&@
zft+`@*NdIR=tlaKtVJ7lp>x{jzQ_|Ib2=){Vega_dEvs_3lEj#pI;uZLmAIrtt)@A
z$b(v}1p)Er4&I}PXMxwY{@H=irS-&Bk{ZoYDLg~CyV{Wv`k^RzQPor|v}gtC026Ao
zfLZ{}_kbedhz%?=r*U|xCJP61KGQZv;pTCxmH=U3c8x9Dp@CJp>#9|wuBwV?;}mCZ
ze4ANri@5k88x(P=HQ>-Ou3z)0Zm}yUn|Qrf9QZ-P3H$HHZ>M3y_jiNSd|r9{dELph
z6-+{cwh?Aub2Q@fY#2I0{d_uyi$cE02+WO;n%Q4`V=(Jw0TRM%+v}gCEx5m@m5=D#
zYEy9(aD7sRLEYI!5F#ru)}Fp!J~JHraaT%Gq}e=gCZJhV=!>}ZT`pAWa<at2%9VsZ
zwhucg;tqCOlxv{v^(*A=9{2l}Y1%Tvtj&om0hW>;Kxv-+bXr52d17P~svCBBh17kU
z@|D@RiNj4Bs@ooep+AY?Jg@w>xGBe>KzEaTnu-hUz(gudx0!z`V!x6Pg~SZU-bxi_
zM%gtQk{6bw|M1z@<9;E$a*Cz6J)xXRU7v?tTx`%-KBgfV-s-zPn^zmyM$%yIe#z%}
zFhi%1U*-fkM~Y9F29p|1PEamzEm_AYdak-3)tZ#<^1IS<3UhVC>f;lag#Pma3pF`A
zkvr29JjGaOUpWYLV!w2<z9aN~nZk+&P515}Qj6=6EjhGm0bCb{0*Z(y5F9cUBXe)i
z(wj9NCnn?~n3w((-*3A5_^TLjRbLg#isyCVrw9hmBUOcK6Ca6$Sq1Ak0C-n;n1p&~
zM%|y+)>UD1QI3O*Mfb=X6zyTf0+YX7XHDU=<jt8cmMojC4y3mQYnLeYvM5)!U(xwZ
zQPYuKy`)!-SuS?x6)kAsdLJL0ohGjP;xZkY-!7{Kb}~tUx&9hM5V^{C+PW7_r6E)x
zSX^-kIeK?~w*UOIrD?vS`XkY{o2-{ad#+kW*gED*cc-B2wYg+Wrc!c%tF<paIA^o<
zKHWNyAy=Awwny{TJ;!2@O~Lr+NMgOO1snO^<uc|_dEqHL<~w~fLqKXJv9V;}u*oX|
zHY;@RPD4zmHOvMG%m~xq`|sS7-D7xv#%mCe-1;q28bQ5av-AnB+UD<gZR9NU8wT7}
z=r^NltCwh2rX=(juJsJ5SEl~veio|9s>&Kyp6s$9|7TodK~GH>wY%XyBK1tp7uzqO
zC>lnG{UUI(sWswO(~B!Fi2t$JkDu&8ALJ+Lm>@rr&g9z&?d_L49KRw<?dzQyofN?;
zeDq9JB{Nz{aSAQqN!iPoRhz8X`8w!8kR{tFS#|zNGnxMp|DV5Al4o5K^1!d&=r~Sz
zqb|!Hadtc;vTU>q>|Vmow#6aKR_`9&D{u<aw!f+6YapF{Ps#J<PYsNC0@2zXj&0bj
za0ERUjDs$yK@rpZWh-psU@T5_Ryp6mr2B6XCMJNP(~#9u6ee2iZ#&yF@l`1bS+<G3
z#etH!(k3d)F`s9@S<1HIY1hdl<Z*D-3p&+dEv@#TxVx^U@ZUmChFxF|d5!zOC7K=X
zMf1*GRXVhw{mJNuzt!UGI2g_XzBznE%s9fm;WESD5;Wi;q4$H|l$mm1$4;a3z&MfN
zZ}G_^;G$#VNukobBcWLRp93G8I7bASbD=<43_f9e8?yYj@}(vM|LVVoTm+@|zm@5I
z&i@`3{r~@Al*g=WpnjsxBFUTCJ23-4|645hRZ^1>z6MZ<_@Q!@BsLvNju--QbrF<5
z9ZBFs%m;wO&LU*pp-=+wDE#ywWJfN<Q1kFXzIceL+zGmZxbQPZ1rpO$=rmBm-hnuX
zNoY$}97J_h#OKejw{^~JA6#TwIt8bTjt4qODLj%jO#PN8FnsfdA^<e%EPK-P+<$T7
zgs7>ocO{z+e4Pt8gMX;ESyBhYZZ+@7ot?=o?(%I4rn0#x9*tcbO{Cy`Yc7Ud9g{P1
z(>+?A$w;0?@HnfGm^#q`Z6Y{ic`UBXiVqd|a*NyH6vN#4$aDI+&dk9Gjh;n@LFaqV
zsb@N#t2>W8r=E+!wvL@fwU!@!&aiKKDO92)2^HYk5k2@#^kp87#VtKOqV%M!prT;v
zR^EAI7+;7y9+dw%RKLkRrr+dfG6xt;{9(5;IH)N6=oM?*w~^@UG@EJb-|;kfIpy!~
zc*ub|2i0Vk$BB`y&dlZrt{?BfoG`p!ooDBB=o_DY?&Yp`@(Z(8xF+k`n|=Tj&P43r
z|6KkK7$?~lp?Qmv(9rhH8HZ-2La>b<4Q35y27!B=3>f*?36+n>S!vM3TCa6;c0OP}
z+YhmL<`X-eShxGt>1Z)A3SmgX5#C&J6|Ej%3=D6JR&pnLN6<Df5)3be20y6By?sto
za8v7{&4~{vvWGj}_*RNLnt#I*I?J6QIyDZgmz9Q&Em!GSN(>~AF~Loa15mxiGv;Rg
zU_wA0?g?@?aZoJ9I9t98j80dFRc}CedQiomJ%gsY9w!7qc4gnOUv9uG`M3PvyJvxJ
z<!7niN#7hQYHQ6<zr9^^cE%O9Je)z>yT<C`S-Bn=Y@0E)O68uQy<b`S&j#q?Z`*)P
zBOe%*X^R}Kp4Lv#fBg*yM29*ru)yWdH-KVL-}s<vG#-*$pwS;1%)BV$I^PI$d|{^V
zdO+R5!9$HYVs@`LU^;@{>X<LE@FhZC(KHYY(%4aU?^-%ge+A*?^Xo0h0ut=`mfs)`
z2AMXJs2g_(>dzf{QP|icXv`=sWwa&>GQUA{7MQLrAQh?ztjxd2(-6%9od3xpAb|PG
zBr<q{Iuvw&yRBC1+|sZR)w~Ct*PRet&qYxqu({NdY#PM0`g@uPTUL)r+Q)<9hTr6<
zQ@7Kfsw(fe-iA__VG)#3b=qCS>Ey@Z&s%2)_C^Ep>jZ0DAWTQ%u#Q3A6I5O|-nrUF
zEC?f43R*zMpc3zb#_|5EhyXBB%`An(O~~z82@3p^V9Hm=QrnKesJH7L(XrbES&>*=
z`sDFOg^n-58<nn8<rLeHGIP27tbP?qUnTMZv2U<<0!62~AWvIW&t_xec#<ciYJru$
zII@yHuBI;1p{>+K?aDq>(Jult)sLC4SnX(@6nj*&!$CD3o6+tP6LdNI?$;KZJCLe_
zV>GE+xmIv03(`VM$D?EUUOvtc?n$89JdM}LIm*Q-EZ?~a*;~E|%A35S{VsL|#M#Sk
zM)$d7P1yR+rcN^BEk9_Wh_2(e`vEHk)3}7&Z!s4}iC+1Cx;V5y3#%i<MFAZO2d-6t
zPeR*2e7TxHFwl@ug!#^rndBHVX0|lH)QsoR;6yv=lGt)D+WL#5+aaVCL;)I+$9%N&
zdxIkD=V4gJ0>fLWEYDZ5&=N8i5dNcF#B@4{Y&TH(yipx;kIAZ8;1F8b<0W26VNp|q
zD?GJdoC2Pt*D@??hRAZ;p<Zns-)fZa3~Tn(*WvWIxQBYxu~_FGpjO^<AN}RyTBMo^
ztgCAR^ZBJd%PG4NngebVqv@B-46E-kuNcBxcjcB(Flx-uC=fwV!)^wbZ3|XKPK|Di
z^Wf{s4ZXk6c)E?yC*s{D%MT<^jL5BG^<f3u4uv(Vb?3ALe4F03M;jR%x1(gXUsay4
zN5yNfADL)i=OftkU~6<6xqoXilu=exNwv36SF1ExJ0z({3G6Nc&{l0ENAIYWIz1G<
zV^v%)r%tSst%pVgNF!Dq8j<tNGcTGHybh>dP4AbL$AiCm-)7$5?JkV~U|8Mjfpj-7
zP!jVycnqcnHRtW56-+^_xgsF)j<kHO2g6z&Y_Az@uS?XExjV4%8tvIo61cie0)qd3
zm4-JoD@6+J($)YTzF@~&D2qCP!%B?uTHtbhV{6v>v5N~g$53A}{b)D6d<$@q?2f=p
zQbS@7HbnwAxf1^uN09DW`<{n%CBI_sJp>~24MI!<cW@`M?AAM}Pg@=m!8vyhnWNJO
zm83p(O0Se<Hl+65xt{?P&QdQPGP<|$8eaQ(X$O$=69g_kb7aS3<7bd}5(2*H^f#@Y
zfM>IGq~#?9KCDd8W^xCzfWF|$K4$p3P^$)8ud87g*$_Q6i#ZwG=RpJaI{q;7kYFVt
zWEnJGb>tkHNV5D`s}2CEHiLR^>;p32FAtVN^*1SopsAK82#o#umUOHRax_w<B!7T=
zOKNAsy{8!~<|v9)ZLdN>sYzR-R6RoPKZ@+%u^-tVbvdfT=<2bP2Q|wwd%>+GUmv9R
zabw4i3*F4Ux4Q{cn(sCg0T0|r)x);Eq4E@f$Fi`1JiS2K>gg*RYfjc=%j|Thh>}*V
zDZ>YnrN+%AizEI0&x=`(%89$&&s^N~wr|bWjDI@UK3i*0>7vtE#D5;#yv2vb3q6%d
zLD<h1%$!2C<82RUD!}a&jK4puW1l7k?$kFkq<b@FdzN(*p;GI=%uXdab65e4>BEjz
z+bs+Ql$Vyu0oCW?uW*Zf=O-maNp_GM=?;ioj-IJE)9nxJrWU@q5;@z8lCYke-dut}
zbl90zWApDK<70!(AQ%Ido1&2i9zMFzu!}BWhL<i;g#c2WM9g-YzGxq-R%fzhu&F_1
zgiCMpD_-V#F7s<(uc)kH`2N9Q##ssFdnF<od9T|Q4KCnHdIuqNIJO&smZmW!o`c{?
zAV22ID|xo(b_&$`9LIM&)JFw?o~Nz(&){2J4dgY9xFtYP1Zy^^#-6>`t;tLS&gpvv
z5b$67rY&7;2x-GxjW@>{hyZ5O9_i2Fghv;NS+>HfzIA~R<8$C5Ep(c6Y#Z*7XQp=I
zkz~L$Z}&L(6u-a4##nAy8ch6Jjh5$nY~=oka%T&HfW|VdTcj)RuS)vM6LC4tGW@Se
z2phe?)RXV<`J}|!b&JyOiA>+T3^Jh6l|B`X-2?MedOh6zY*a>3iPPNn{g#>T*rCyM
zazA~xV?S?52eQp5jdm0?T{E0jpf{MP2(+kpknaoHTKGRIKZ4bhjTEtVPn-Bh_@9x`
z<GUM`^~S(^UfN|Ux$cYFgxvmmw<~9zp|pGUwYutjpWvqPFIUj8UIRAQw)zd&>3m_)
zVvKv7X#R;x&+-D_xp8ejY><u?EsRPb2Lg#C2aD2YVM_dmoGWgxK23GMC~LiDJG9-t
ziIIeRnFSOJOPgPjw?7qqLfr(mIzAM{#c0M^)8>ss`xD4vp2Aarcs%IX+QNep*o`n6
zD9xL?I&ZCMe8T@Kect^hYdz_rsSnxG^?5d{%PQeBS<Vxi&4>0h%-bxDJF#F<x9GHe
z`Fn1A{U1z_fd`aOPQQ<kMHnV;vc2WdC-=ZN{&@blAaq%vY;2BcE}6G9)XFdL_wY>~
zdCMY>@A~acz*Q^~Cr2X(R=;7Q1Y#=J>M2#PD!l4pF$AA-&75tx;WpHf_k=E(^r1oW
z1+}rxucut0nwNT&=FGan+HZWf6K6_cNyosBcy1kyX(Z`!=VWwr=V?wqT1ipjSDg?r
zvLRvZSDsM!UKm!J3PHGDii#HdlXy^R1|*w}z~dLx$?K&VRo1+gw9BTvL8N5fjj!PE
zOqrvSf;CgOUse{sWJhxsI7uf4|49Ea^GAjLUeBpAqMtt}$@W#|vTH(AUH>}Bq~yD^
z458gFsm{F0KLO}FF&De_@#Hg&n+Cau6o)538#QZ|ZQ=Qf$S?SU*(`I~{d%r7SU2CG
zVtZaWi&E(fsy{(jZThlvvKYbmik%~5nu|o*#+IJMUCwmu^6PzJI{7B|*oiwQ)_hmm
zRJB#Vjq#l1DSik^AL7H?daw8YVD7!6;oARy{}f4x2;xLZbV8yA(Faiy5`;9O2N8tP
z%Zz%WcgaEYQIe2E3&ALZV3;6`-bNQC7>v#+_r2wO@A}+(*ShPs*6;Vb>;896CS=dv
z@BMzi%Jcbnog3eLI4#8=;A)7B;%`=qiQ_Yb6F`m<iutY$Gl~-U*2GO)BKw-347C~3
ze*o><YP9n##hA5wLSb<W2{ti%0VySlxO&=X^lnO?3T2PT^aG$l)U|k>G#_&PI^D|k
zEE1^eS6vJ`Yd*<*)__h3lOrH&IGKv@Xscd4coRYu&}kl&S8u>`AMdBqBgIaAy7V=s
zse}FwA8qz0?X$2j?59<lz-n~pSIDC5-YNZAZi^vfM{RTMQXVGwYuL?u(Yz@})nP@~
zb}sBVd~4OI<zuy=<5R-fkv;76j2NOA1=i*xtm9gs;Z`;3!^4#^U5YBN;*-DIo3nO2
zHKUG{qAz=s>X)1$2pYxg&Agc=Or`>!toynt1)vc1w6iLv@NG*XE@>dcM(T5Q-)Pjc
zPNfY4G;T9;xsu|w{F%bY#pcBkEu-OnOuHYgT1hRR_CYa=j{ADQn639_(#qN!fBWua
z=kH0DlLr6^`gLow%l@B;xr4)ND5b4<F!*~&HCFMVhs<PMp8no%#|dTw7?}nUf_3F=
zwc3AmYAB$sAv4_CGofl~D4n*>6J<&*YdSvb?uqWKu&mFIb-AdKhs=b|kv#%;jxcw%
zrCW92tn^FLKzHd5nsX__+S3Gqjm|}h7UQMrPS*_;&KVfUstP?%b_@3o7}{Az3#M-w
zd19RLFb2FUC0?gF9T8IB2Djs)J8K@89>9#Dl1Bj=)eU+))`kJEHo-Y%zz&Axw!Bd<
zmyEeJa!DE?ZZRSl8}^u-M0YOklykSr-IQ~onX_<+<Gc9l*9eF`YA*J-s@bPJXV#%4
z{iMzonYv-b%1L|8e6c=pOV_mSA`S5y0OSpn>+=1+mpUOX=0~(M%Pam$VUWlF)^SZ-
z_8BFe?bfye9#g=_?ruj^m4Ol@)v3y0E17}poN#NlA0GZ>E5AaJ1Lfte#xHxKKbaWK
zGS-|)^l(zxcQ+AMG3HmvzIxgh48%!*a<t8J8n@}#MqLSwiNhL7+}IRAf(qc^wL@7C
zXem;-O0Z*QlPxhvkU(`8gq*sLaX7x3%C^SSqbAs>7a-O{qwRc%AC5%ON~^sqn<M15
zR)tAe7=rQ2>}zH7S8AVYo2UhQf0+5cu^~oS*LkC?4+fa8y=Wht_OUEElQE~7!aNT6
zhzs1MvKKa-r_sgwS4PEIio#8^UYRERmY#`LW2N-t5VmjRegeCd8$f!R6mll}%_UoD
z{I~JgB<tq)%dp?{vlN(HNWu)!Gkd97KIW{|YHH-1Y|tG4^ds0+HzWh%G=^^cgmvcD
zD#~aS!+c&HL09ac>DeTq3_5Z`vD2|iwIMGkwQuUGvWnODVE!f+aI`8XtzBmj2>MI;
z9v%Je6&i}fX{1DNpBit-Am>k`JI~OBd9^PsZA5LFF^eq(jma%qPWqm;C7kT`qRP^`
zOe(XM5T>7zo{yCwtUSKxUegCli!Q3tF36^)xZ9%Bzi5%IZ;F-;cVfsDU%M{v30M*N
z4Dap)h^N9v^?)31;9Z!$2(b)w>z~rv`)B(=544)_y{+%3uvuKO)N_nQVXU8xbLP=1
zH??*b6Cy#u*d~C<Q{-3UE=Bo0LM26+D>6;kW&;i{)Jeve*p1n#eD{o0&u$uhz4bj)
z@h-MaU8u$<ES_&In7I0d6wxLsh0HJ84zhm~KDX&r^kw?>*u(@O)FO?nHJ|Z;`SAwr
z^fg+Ew+lt`e$Cbgx^;Fd_xo|-4@j-TT8%;VkxR-dQ*I!qV?94@6KOm4-0~mNP}2Nz
ziddmHuBZ!xZ|1eVqKVj%Wf8g<-Q~yBtiMZF(=2{4>KHQb?s<0zE)mO|8fBGnv_F?8
zonoVBJ)O57b*?bFcw9!{LOb*4Wgzc>DX}C@v7D9Uz7u79b@f7r8tVLwG?h!DXXp7)
z;U*}&)Zg1ZM2j`f=O5b}Pn|A*J*xN{bih!TPD>PVIP!`WMPa02eP^8lh4E3Z4J@ye
zjtV^Sn+luP=P)&A_~P0`D?bpoz{l?{7EZ57l(e-V-?bC9k~B|xJ{-g&YGEkbr<(rZ
z2AqL`_RW@aVTF=A)7IoOv?`T&T!P&7qV+4V)CC8X62$KiAG!gxti|_nm1i_JhQ>v=
zUF$)nb=sdDv2aE7iT8;FgTLoR_f)(;l_{jndA;&J6*PxJ51q<s;;=(|6e`YK3~?=s
zz7uv1KTQ5ebTqXlen~g7S2)SFHC*N~iXq<iUrC{3AaA4(x<YazG85sjZ%e>?)VP1O
zs>*+wcy)8rz42@7m~<le<|BZtG{qtur2HdLOwA1LsXi<yLy({@FH1V`s2vZcYD%X&
z7#ACLM%Sp6rsg)K6M3R-yW>M|wO?+n|F%?gIAR+9+jFV<%!f1i8?4r5g5GwnRsl_4
z<O62uc1x+c0~;2`mpQ#O22}<R-?p--Bzdr)r&!?6=d32qY+SSoUI;n48X~oF0%1tF
zP0%Tdou<h=Ye18|c6Mua<{EDe*L#J(Z6+?)ntyk&>eSsrGQauM^&Ea8v345qoPgvz
z!}PtfeVS`j@7(Vju_tR2tlpHrV$zcx_I__9U+U&7%{4TxtuOoQh7zF{CpPM;E)aLi
z$e6SD{ei@p)e~i&&va;_fX+O~q^^*@e7H<OsMSy=+HOD^e(h}OfxdEsrOMpG9L9DT
zcK!W&neTw%&LfE#7vWNWhL!d(c)&PuXVBFVuLqmv&<P)(KH3_WykJ@2TPSF^A}<$X
z;8?fV_K*n-dX)`uV!u!RtT@gWdieK>x97escd5VD)kLyo!pgyPsC=Q0=C_$~?f2x$
z=wjh^2PD~Qg`Lp-wS5OG+n_2kVhfnMg>~xet6c!UNS9+0MKi4jdrz};ws`XF=;YZS
z#eY?O<T`k#P8b8?Y`lgQGyDVx9O&LWjXbC36Wwp!O?BEMhsvYnvo>N`eveEp^eE)i
z9aX-(McOxJdH6F@_^;Wal<`E}?5&Fg@O{973H-Zu8kNg9;Ykx>nckzr4@;L)q;hnX
z{&Bh!HbmdS1@jjZiLwJfo_!J#9G*fI{2+)_-+LW}jL{xB=!^^(Lw^nKSiO(SxtRK7
ziUqs+Y(9e{1UL9VsM=1VOyL)+O46DdOiA<B3H)<^Z{`fG=qXt#I01y>pfJs+1m0pd
zP7A&b)3Imvj|?H9R+K1IXq~9nUa{ye4o{)u<8L^DMJ>7p!mvQ&HCf3I3by|5@uWkf
zU6^#m>e-N+m0W4-wG*HMt^~wiomrdm+;TxTl)Yyx>qLQ+EJ^PF$0GQA=iVqZo^u%t
z7N+BqAN2!8YV(DEFOgnC-`M%iF>ndk`ZyY1%m4uPbx4>1mAxcpgXtP%xc+<_0L6Yf
z1C&bwz~28H5;&<Q<%3!%&vC0S@Kv4Ce|?dx(>fDCRiJNCe8UQmvj=tS7PoPrj{TV9
z&z*EzdS-ur2pnJRkRon63@WPwaam639iWSA*x{Z+GNalsxgLD_1g%0h+<mO8{e7c9
zx4pRrp+^^o2VU8S9N$?qUxzfFh6za79O5sxh8=F*(?+xT^w-95k2_Qew|~xR=RsLM
z9)`LK=00K$K<rPeu%YYwaqzgr1!ak>K=CyLAPo5gGLE(tjM94wDQ{fBcT>u9<Kw?_
zgswY)ACG8c2U-y8rHm&w(;jU5HNa@w3VHo6f|0w{369bxaG#%IxOl)%ukgSYyM1Z_
zyXP(tXNdsFc19~0`)hm%L<WhFUADZj*TSz9lmK0QPZT%~Lnctc+%%W_{`yO@<={@J
zFq=hbl15;ekP@U7NGBFb1}!2Jp!$s!fZ+;4s*6`ww0ESk`a4)|n6S@*63ctcl9i|+
zq?Q7tDa{Z}R%802x4)56QNPT=N@L&0jDDxmX~baoASB6ofEhjDK|0|K&t|Ve5+%r$
z9J=MrL<AayXMjkPF>HEJYB(CIiH5tf7h^!1y7!<KK*1)hK$<K7Q1Il=;sOz#88PGG
zxPUI~IzTh&Lw46=i#Q-SRrrD8%D71f2-gp@;zsvRK!Umf&qvZ^aK|4rK70P!{7g+1
zsFr!nfI7wEN`wNA@pv6jUwJVCQp81Qlr6HJ+EmvAkeR)NihX7h^xuKSSr>8D^igm%
zpiwW#d;8V_5zm(sPj>uLx_5v=&}3{QM>+veLCPE?l5H=Lp?2s(-@6<P$jHi5#(>&N
zKHl^QL8JMjbfx3>>}UhW2TOm3po5Znq_EB|RNTA{V#S3wh;lQCfNLVGD$Rit%yp%D
zLN+9Ov^jw?*Z>iJ1QoFbnxW~vdw5m*xbmagT!k`qO`O@Le#qN3dy8#oj3&fwd?k-Q
zD(N3CFiHgFz4Ali*vU69<+pND9llsCiZGX=o9@&doVB_|!t_4U1W5H6!7|uFmJi3_
zYUpCNwn8;u!R$SbmJLgHtGyYgq8|(~A8H{^bx(Pv9#m2@2`hz-NEW|&;y<8X?V(+R
z0#K4Y1Mf0;H%Euj=y1x7JTKhca;d*x<xbiRR5?N_)0?$7Lt_L<BiK<$p;yvyUho3)
zHe8yjGX3zMM9haYfKz|^;@F@M#sgjllqk1k0Bz61>CRLIXkCG(rCsLfNCzd?Jn8tk
zO-T1gb7R5^!aCR~n(cKbwt@uaX%9~?GJnP$ORxTfv<9XcSLKKSl}oSgrSb9A5b54&
zDmSDZj7C)L1tc4T3fH?p-6+euSMBbxzg)dyia#ex1S5I7N}xK197z2bg6a-HhwabE
zYP6kT*4Y&#-uo3x{30YOx~!-DMpNfd7#{G%8v;wI5}hHxk%t!;il9%Jgg_&X-UPyz
zd3`3N@3Lz~-X_=BuOp!P$(#LL0E+UCy<W%2wBI115H0xqN*-Pc-k<)eHcP<H5e0QH
zN8wmGHfSQZ@2TU_(U9_SQ8FE<2JI@3TQG~+ti#GiWwqTaN(`W?un)vdnPbHH6mgJf
zxl3?KwkiQHUHmb$;slK~XyTX=aX_zRMHJi{gb4gxuk-x#OIigaeaVOeVPhe~5%Xj)
z5&YG77voR1lLKS!2&h<0FwuzHFph)9HEf;n?JINEuixzLU_8sn^OFQ&(zT7)Hrk+>
z0b8kqKRgyb=k`mv>)$bQ9|H8EHY)zKn{M#xKkry1RJQr_9oq~kECoJQkr63Wp?oTo
zXHUiKxJIUGkUK*ap9CAXHJULnWs%;YF&_`BjR{mXV?-U_g~u~sd_g`+hJlys6*p#Y
zAun$`@x;T6+1(^Q9rg)mI5h@{e}v5Tr<n4rU4TZnz#?eiO%hjY0>19I0wD9Wfe{I_
zaxET<YPX=q$P8HcR+|)UanE?u`oZQaw_9~nqW#w6UC_`mJ~B$aMcD%(B}_j7phex-
zls@0o+T|j|9Uwu;4l@bJGg*a%tM*^(&z1={y=C}e;hp-4|146*{Ze-1>QmEzn51hC
z3CV-QKqK@D8P)<R)j&sXSF?aYj06eKOlfBbfESh*N?<!h6#Q!rng?F6NDK6|$8D+z
z``N+%<{+q~qj%;IOgk|m^loxqFv%-+>u3(bev|r50BuInmG1iZ$GgG06&(wts)3rq
zt;O21swFUupbQ^r&Lo9>U$9aLCL36VUwijSNyZoJ-pynp?P);yiZ~AIXJpwUfGNwH
z3q;TK54mjV?{z2%Z^~Uw?0#3(7Ut=4-d$ScDZHv&(9v26_&uJH9E&XUOJ2_|P#wJ5
zamadME|d1>$g}_(XL3p>)zmm3oivVR^Z)b!HBo^}h<QX{(h4R;MSg^Dg%NdvS20Z!
zOMf+p?`=JUAcs{+>`exj+fkR)v2xC~Q@q!ynC^ZBS5s6p6^yKS=oKXMb{y8#%_UBy
zVv*okoS<EceKTm75$+dvGj<!0^GthYXr4Ok(mmy~U>^cx#R_&p19zA>ZOx69BEQYN
z;SNlswr-jKWv%?SMW(u)JDVS*L5FZSMn464B04vP-WU|0rw1KShoh2{cXh=h*{SK=
z<3;H|k9G6!hd&9(?(6fUrsJcrrC$_=Dc3T})h_-lY}YujR?&(=lt<6KGvKV`>6ScI
zu~PhU!J@v`BEBrNDPe0sX@siFqurQ)>{t0sCwrqOH>sI??<*uJ78HzJdp_JU`j;=k
z*^jy%W^<h)*msH{<5T39!b(falZ;nKH<Sk^p_Yh={mvp<>d}w0k&rfl)KtNM7w7c}
zzj&&W{5|ayX{)kbJBf}8yN>riT$lU&n$r(VM=%r|62|yqR4tsfExNCDunCU!7I*VM
zfV}<!76_SeMcnncX{0WjP6eGnkbts)F9WaG`<RJ`2^?G(=0t$}S~5O6A^Pi-EZ?I(
zoK(O+`l*Qtr03AM-vV~0?D$SSMKo!q?ft!t`x$fTOsUVGvY%=nXKbD-Dv^)XPqfB_
z@75y9W$5%Ky{Om*q0&TU%=8&gA;-@!q&NWy+ix9YywRG?_KSx#m>+GVZ(9VqJDbKT
zYRO@)RxSnc^$j0`SED{Y-b!XB<ttGZJujrvu^b3eD6OV2dCb&Mm25|hIZyHd7?9pG
zx<+AQ>{Bc+ld?q0OT|b&fOtehoU`qhYH$2)sG%c64@9OJ!A`fnWp-^RoqIScIiKQ1
zA5ct{ddLI2SJyYNx!xrNVoJJZCQxe(l;Px~vP7w8zekEuQKCsTCto9+E|3PXt_;|~
z2y65=Lh;(QAXc6i-=$6^+_nX>TYK8)^a5M-^4*@1X$Yx@mwu=nLKKe01)s6ID5E$y
z+PHxgyL<m}qaY4YUk<tO60v0mF^L!PAJ#sfbCfR5cPMAr!sfKpAW^BA-7&2nebz4T
zanE|o;J=k<j29e%V4>QpHr#gLHi4nd)OLVfuID0V>8C1+k)4<#NfJ7MxQP%arwmq5
z*kC_$_>v-5yv19Fq_v-?UWdQX3U-H3w(90MQLV;P(Ilz!==qe2m!7nHIU<FYk%3P=
zh`gAlvf{5KOS>H3O%7jE4(IKUi^%@%GVS#j)gj&T5NBY!7O`Ag-eCdUz6a5ij$@BJ
zO~bZ=;1?YBK8~l!^D+Rv&%d#v7VCA&uBk0?$D3XJvq{p4bB|7j)1y<7x~T}pR?b4C
z3~9+jP{)=5A2Tz#3~p#CmBJ)tgDApVf%R^wTx~dfs$}%#N`!SlXl7YPV#54WYfzbj
zwT`-c7NpHeGPihlefFA9SET8aesILg?OrmPdKE6lMXho@$Y|9xB|csAmz{pYQ_L-r
zmfDTeCzoWH-touhR)-Jk*DO4J`a)8tXb;J$<J+suDs^%h27g}N!PBqD90h9<ZtFp6
z$(qR1dcNvmT+gp%zZIWr+51$MQ2f(#1{B@Kd_Dg%mS`}&PU9?V$Z;hcwDCGGIM){@
z=(9vn@x2_Cv~etVDu^FfR1(^B(*G{Gl3G=9XhO=bvpLNdb*jM`JIPSGS3aHa)_S-x
zkMX*e{HxSL#5>b<P~DzmJQ`Uuhkal?r{%Y0RQfE5{;1G-{3=`Ot^YbEZ=jSp5!y`7
zT579PUN#-6!U*rb9&iO;xK$hqBI5k>ZHy6J-QQw!<C8$;C1lEm>YYda1~ya9%x{e^
zhpqAIJc9ixP7|++gq5%}Zj4r9k?Tuq;*qg%$Et~7UPQIysn;v8K#Hj8pqmKJ6-y5C
zpsxMRE{#>XIZOoXL&sBxQ=e*NU++BP!^!I>QR~>+sXFqLI5{5LJ#4sYn{kde!Qf!H
zQD?2}@2_>iou!L>9+fjU^3`gn5I9OD&fYYBwb!`X8dKdWirlJCf+U<J(MvwjGa-x^
z0;h|@<ZXE;7MH>^bUCiXfV+xL4Kp8T{u;)8{or|3PTh^VDu%7SKBpX!F&&V>7e+VH
zO_TZ#3nR<ja_dihOuiFNWHep^rl5RxNAs(NiHgG(sD#Ald??>#P)1q<l8LoWPin;?
z)xQRoSsnK@2Ah8s|GQX?NEZSklAK~1!iDv|I%Ivg6Mv5u<2I#1wjG+2!J$`(gAUBW
z8*FphjKb+#=+yFc?GU05khqT@xJRA+YfeyjN9bU(^`(4qyWqSJl4{+C_(EyLd%XdT
z4!^C~kq~N`o;zyD5+5s_@O-T@p<WS5I)Rq5Ck_c4p&!6gUDF5FdRvB(qi%htM?E+S
zMc`)JS0sOlMwgqNb0CWAf%ZXx&W}d38P*nK@2LSvwiZLgJqhk%lA5DLq|D~yIlhu=
zzKq$;ccx0U<*8#D83uNivlqjqt4de)B08BPa|S=4|9sKT)1V@Pb~UWO{Hf0DWVP)s
z>`@k(yf<D=Fq)~xZN#qls~>r5QZsz{=1Opmp4{4ApU;ptPK|mJa!5LGRUekpK*5d3
zu#w6GA`cs(+u`?Sn>a9|&do&{^sYXm{z%GvUwxLH+kTR(_@3RL=sr`TxeRg9{Lv-S
zzUVrcp^yuMd)~SE2*+3OUs*NKkkR9dZ@~Zj{V>GEtkd*87GajfH_ZS0A5D(WyRtvp
zSbmwIkk8cc>qCt@E_2xTCfKVRdNn}`esniu8qo9rl*c<6d0I7Tgju&{Ox)5GltgHu
zk_ecT)EP~eL-8L)hGV_o|D__o|J}8%roCCb3h5e2KY$7h((WNaa?uJ<YL6afQl+mO
zzprNpX;GF->_<}0Eqr<YfnC?ZvP5&rOGxV2;M!jC7hYUxGKisdpmwXw1q|;?47@1!
ziE$Z{<^(TrC`sO4GJ{GgKm~yug7g)y&#^y)diREjhsAiwRxf}^j)(fRR*MDbLU{gT
zt%#DULI^Zz0Ek|#z;|S{zQ6%we;~Ol6BlWc(3uCG3KP?{e-h(?PAdnqU9%z#iM|XV
zAT@}gm3T8R7n*5gkZrfoG@hpG@q2dTPPpwJIF!ZB0K%pah|zUGel_;E?J!5W<}+r|
zWJO>AMCLsdN14d&wvi6DLgRzT-WWSF*$Nuybe!}tn)#4Qfc!fF!s(WiKoLZyz64O3
z-p>HvoRHa015-$=H$Z$PXiE)!NCOrP_ai{tdRz|58j#`c328)Brw%iUVI8m`8!Emx
z2|Ru(_f8gwi8QSP*@%JSR&gv%xCE*jUAtNE$6g0Skai~_R<8@BMFY69ya<&!X#7*3
z>S*`&_7=gh2>0oy)0}L`M2{|1uNs~E2Dm2cpUQoz-2Yhy^SAWwtV2bySgy5%ZSQfi
z>E-yCgnl@BRBw^ncRaK$6*ug>Fbe12bjpV}e(Y0tAiIBUtlFwcg<ALtG$5-=+h7K=
z%2n<I^-S7x10p19dCCapnYn;|La(?fpv$>Ce*@;XUMZKfW7VkY%tL7cVy?_v&@qpC
z(jmh|v$1%q?7=668B2e3;jsiZ?jL7iloBair?<iwrwGOo&H(+vgUg>O2#~8~4CK)9
zkYf-76hNYof$Fl~v9uy$FaHt+i}?=7M~csjPE`Y8PXl!#q~^=<g>vzSEex8J>_8Z_
ziD2*b*yoNEderkfg!`8A9^^1QRso7(u{HbQ%fK~Rxa=8oGX&&2)j3TP0Bfwyv<0H5
z+J5Eq8L0c#=LK|R7t0PQmH}njcnsJI^;QbkmO;$a=Pe|>Ij!CA(7JKBU00V6K%<bG
zuIM*iaXVQPTRS7{_!-WduBSJZa2sxQEeHrPJ3s|k>2Swu9}-fC1ODYe^{KTZ;4#vF
zXYA4<4uk;-P<kY3tjdBI1%i<A{*v`(=&`#4QDD0sVHYUR3dwFIz%*J65x>jQ^+A;q
zsvg8c`D&Rkm1E>FHEh`of~N!B?WL1!&<oSjP{D`?h$n-~iE4knR)3<b3`(BA0X}{I
z<A~3C`%`23z8J3MFogu(3!n*#7kEOo44im1C{HN4UZ?yCjl$c+08S+aGjT@rlh}6v
z|1r3n)(uMOz-1b|3c=n~*;44>GuNb=<mz^sL`4Mizp4y@1=b2`WY4a4Dsq#6ozw2v
zvgGKup1ROA^Oi53svkILOJ{391ML{Dm$Unx<KR<-mZ{t34wMzw)NIjrc0ZD~HF!ez
z5gLS7<D@3>+6Sl#dtE#UddLq2`9Wna=niv^Uhfc=m!D3V>_J+JJg1r6r;aFVYFbnH
zOq^&|4XEuVLNdhJ=_@j3UJWWxSxz8H_+{8{NWNxj#Kj3ck;P&Pj2DCh6SxX5eK=Yq
zOl3Vcw30#u7DZL|HAO~Pp~>-Hr_o96OVh=S1}eju_L+oz$YFZthBEH4)*Jf9hO?9=
z77^<DRoLr}H*{{1f2b=y0)@=fh57`k;(-n_SSw~Vl?@b>&*n&Ot#t`S-un;aiBEzv
zEvt&_J9RO2ShlR)1^v3Jh{vbmc8(wy!s2`9vXfl05Fg%0*{4c(jjuddlMWkHQWCk%
zXEAh~QGcF&o$!tt=ek&bKl@R|4=}LMFqUX`d$T0k)97m6x}E8IO3Co6ygsi>3&Uhc
zT2%>Ta(OtT+=-`*jXz;FHeFl_Y<I}E>eO%gJ3#f%47_;0uS|qYDLtCKwdstO2609J
zq1!e<)c&5A=W#55ZCwO1+65|oMXNc5mF?t^t@0CHe#bAi?|hI4x~7OHMeho@OV1&X
zM=PXf7=K=WS}Tp-9U28db9FV70O^eueq(^T_R9`~_x=#xuiVQ-0qY4c_x<Z(?X><o
zjZbarKpiUY)Y~5nftt)~Ku!E!5*w<xk!8(_ze>1_U*I<59ztS@*g1cR&0Mx-|Ktst
z@kT(qNBhK?GW*ewkj`@rhA>w<C+<g-ft@sBEv-)AH64IK#zB2$s9XLuheu1*5c2RT
z=qP}_v@Yvr%%SA`yQlUfz)sZ<uIfCW(;ob?wB)`S`O;jJ=QGNXr>P#DSz*?Zf4{TV
zhph+54=4`2=7faPb_PK$rBW{Jq!q8~Lkdar4$UH8At|Ad(!i4K#j{rDq1%&02bNFY
zebDx%CU9N~hn?Z7qEsaZxl&%`v<UwOO=~>=8+65PhkwfecIc<ZnsL{8o;L}vvwB*-
zNkz;S_sIwdXF$dG^R6@Q`%oFb384>@?^lfg*Y10HlW2Z-3#b}vBedM<7QP!T?s!yZ
z20x*zQ4yzX>Yr#8e3|4l!v$pSzlWDE`w9DeNPSmgr$3eKM2xh?$uC2*-Dt`>a|-e;
zqGs-VU4xPHV-7ZqD5>zyMlS{nO|anv(V|*}ULqeVxW7Vii!{qHR!GHgMI4-_1Lv-k
z8F^RevV^^aYuv78f4`Y^e7c33j{sayJJ!|v`PXWEfgeg2^&*(>SJ?BZWX|c`YhJ<P
z`%BhJdA2gKw|N(akuh8&l(*QQa~A;yms>!}_5PbMe1lZgl)(_CycJzro_W*k`AE)F
z!BKP{FyjpmD9zF55vfhL7=kb*6pjECrKu}-*GD2tM}{GKDi;|NoTt&Y3;VY4;c(%4
zNQEdW;b}$W8(Ma&ES}c;8*6I}?X3cQY1T%cf;Vcu8Scect6VoFMd-&Lk0elkZs*Vb
z;&)4K6Ab;8#un5(A@^tb$7z_Z?PR7FY#g=^+bUEbfMB2vjA1n)N6WZ-vMK^Bet+rR
zMb+cCfuRw^MiksdMV*ngC!Sm<38*jzU7`D)K^~d=5{eEt1bIiFe(hSq=Aa&nn1~8}
z8gBiDFG?#rVQ{7$Zhkd}X>N&ladevM#mq_wytK4w`O?pr?VFjQw5gnW{6|qiZ~kH!
zo%)-~q9NU8DFl9d6ru+R3^tL=VIP~~?7TH26HQW86ewTeHVJx0rC*uM=?e1M_3U@$
zW2KGvf`qt)9vBRF%&{@zi9W?M!(_0ZxJMKC>}TGm$G=8+z=Pa0^nJr$q1>_wp6(0`
zBDd7F$Us%uve$o6Wo*+3@C5Aa%Qy55tP-jU2#0?T_6VTFRmNu5u>lna$)$YzBzaM}
zgwH3~H&c*PT@J6v-n*3(HW`C1O2s=_m89qPt9q!&?$OLM&&fC6;Sx%fuE>yHiu0$&
z&Cd;Vuc5Cta|+vT^<4d&&&XEp-uA`5RmK>g2lkic%Z!j|J)(ic(KQ*IA9D2vq!d@B
z)vNNpoli|;TXfa!Vv#Ggt1SUK1?l)CL7qZT8W!8S!8%-wP1;cS>xU*1^2Ea#<Qz-)
z@5=efEcMv<at(hBCodm%b+f_8oouz|)0V{;9``Tr7HC)DXa`USYo#kO=$p>cw+C@h
zJE-x+{<V3dTH>JJpWQF+ilL=(#9X-YYon;bn-PfXxqikHliXv%L>eyBcARGV*C^<%
zfHA>}uo8b&cvmZF9n4b;vsVR|Y{XI>?+}+@D^LdzvJ5dmiOBJP{l7zM9RCWg3lMpO
zL3z&_{K}ieo9~bcpfV-q0vZ5`;E+5<ZuHR2ZXc#8v4Siq>Ma0gd`bh3ai~uUT}8vK
zzme(R+0^{+!a{G}-T$%g3u05q?R1mo`}$8=sj$N+kd{T!;pP1QwT?GJz9Nt(LJZ7f
zUlw3%14-qJ|MFqkwsE6!IT?&P!WBee+y7EGq~HJy_w9Bq=%+p_F!nd&z{Pw#SvBW=
z6m!|?aWgbU2QqFjJPo-~4`nEA^P=p)kP}|$Iz5C^x)Jt=)nPvjsz%{zD2R8k3)(Y=
z&}16JmP0Wb6(=A<7l1UJNZ^pjDE8-N^@ci{U}SLd2rwsDzWVNQ2+df9AQcICKvb{-
zmXbUsr_vHdrt5ex{M8WBox6a6qVdpv?U4!1ip>me(STU{_(7?UEKgw7wZju&+Qh=j
z7o`u7-D7ia6f`IRdjo1;xpd#-Pk?67iYQij0D(b&IrO1jb@Lwz2?8KTG!V|TvcKM}
z1u$VUBQ6z*{N{My-M+4g@q<L%8L3Re;~r7d<ni!j$n<UnHCDlkZ={f$KDS~My<ptn
z4T_gS&_CP;O*n-7_=!+U1)65wI6DN#`2l)R_3j3GcR9ebP14wK^cL$8&@5MA7M>Of
zKqoX3>MOkjD34)N(7s&MH4J=rIcOW|n?i%j1^B9WweJC6@IoDE>h;xbfH>CErdsji
z_)h>3umVux>(Ka`WBI@TGhculZETG+-N=UM2ni5z*@g^tiV5A75bzi|x@f8ojr#WN
zh&tx1NHHGY0(WRmv^ba%lmK-<yt1mfbv`;)OolA%ZXV-CpfK|?9?B9gH^SaSA>yXo
zC4@*bc#;nx_CMF_d6R%qxiRpbZq^M~&~>wS#4SA&0?vb{ZZ{Iw1n|)0qM9UZ9qfZU
z@Iei@T-RwsCp=7E(hq_B?h)M%-h|mfJ?nAg6jH+%ou0H_Gvkh9^MnaU^P!ZsEAf3U
zt&edKgS`#f!iQhRbTM{>mfqDglkX636S9I7V*_^;hs`^0rrvAfoI(P})0ctPY-oJ;
zY^TQo8EOj2f>dD9bHE;=C<D54Yp0kN2cper>=Ba&m_hD;@KAgg588)TfTR9}ODxLM
z8%()o?QdlJgBfXncr>5rUduqexA!q}U%cF6K|abw%U>kAGNvWb{^U55f*ef1$I+^q
z9h$fbcl0*8Tn=l5MM4vvH{<X3ARz$yLIDC-JI%K8>6C6Q=0}VXU@;ueuWBMgasK+!
zyN9WCtx%30YPz89LqD!oS{mkIHJuzk2W0@FpDbyCLcp;3slG|sPZ00TfcQ;y7m9$7
zY03v{)gV#J%Z>9)lsWE>w-p&fT}OpbswNxT&w{ZCX`heb$FPo^f~AF8;1!Dp(1<FB
zY2j@HgUN637y^V%hm0S?wcSdq-%s1ggEpMo3}}e^!mZg>LH2wDYBwI>Zy%h&{jyM*
zGSCPXM}Zt1noT1Fa2*4oTj`+BI|DV=6mo_jQA3gHfFhKFx`r=F*ZkI2oiZT*yxDkR
z)3aNP<Ljrde@FaTjvin+Qc}JFPKb{Yh5<+jz`?Fe2ycK8?OqokM2~=;|7M<90>laF
zd)$sV8p$39QGwkq@~rO>lr7iffD1?GQT7h1sD`-`B!5B4bxl;zWqP4f<xhb=)x-(1
zs2?fv*srq8KO5dW8|<On4^Z3!D&hCx5cH_O<SX<$%CxX+AaOCl6M#P?Ph;jJpAD4{
z`(rYYHq<@I;xjd<tOLc6KDkvjhN*7u3m^MneFgd2u=Ng!fM+QjU<6Vd)XGa?s;|Ia
z(tP45FLsaB=#7p4RApJl#tKM<7Yxd77VG#eX#HBRdUbT;aw~9*l}~&{0E?R9<2q<^
zxefz<i=nbmwg}S{y|ofC3Gn<MBh0C}How{Y_ClGQ9@~C7tYh}q`M>@w(Iv1%@BX@m
zF$YwYJlYc_P$n4>4`tuyQI0;-S7)HT<8qIH=#$$&Kc71qgodyOr<X|3dXaLZbQT~2
zcMs2{kJ>`gB&~_^if2e9-P?*w54s$mNBe=!-&1G2>OPb+g~VLvf2iYj01XBs&Z``g
zHrOpx;wTxV>ZeD?{J3~kDvzC8ay8%vu?a-3o}U!#e&mN+{j~ODog%XDvS}C7uCD0R
z4%G37%g^k5Kc}dxFbHz@2PV9$QAY7}-3td$-N6eR-hA9RF%F+CzxgKd%^A<L@ntjD
zFsivLW&<rvKu&n7so_VNCFAMQuQcxTIz^5Z4pV(597-}dY{C<U7hKn2cA+l|@la-#
z64R6-<v|0Ma!;1!ALtYf@Ge_^K;8_dpc7F!LurN5F@4Z%)Y_)?OExrY3>0*}oUniu
zTRf(MH7o;JOVQM${BM?2B2$Jo8fL(qcPwBi?YQPbB@pnL@O}6Lm3D)^DMhZE^w5jA
zVGryr<-4D<`$DaUQrQr))!D{!TkCfjx+yB%eytbB`sPFGg1)#KpbpT3xJ`PbFHu2d
zZlQ}53uRCa_$C(0K636vF?298s9d3RPQS;(S_DVD4UVD1t7Row{|=nzN5umAbS^TP
znEKCw6)#X?)SP)1$v<byq}PoF;Gbn9CK*iA&&{p=3#SYeFn8`;nd)oGHHx=YMK0vF
z3z;Gc%*Cc1njrT`snQ>6!vwU7D49_Yi<{M`f=<#UVA_Is9XYt}&(n)xQ*Q>&A^|N$
zGkO3kL67Cpf~}9l@S&R2!IFW{*mbZri;$05S{+s@6tJY`(`~MThZW(P0WtjgEJ75&
z`m_f;HZLn?#i(5D?efTeP*sc8$BrVVqc=rYDpq4AG1IzI=2wMZd_XRK{5bvW$tuj-
zl|n2U$`QPDYob-31#CG?VZyCZ>mVL+nb*EvE{%7qyAT)o7Aw((|K@OV%5d6B)Z4(M
z^O=v>0NzNZYF2%-fvNUuc{svjKvy9v8sn^51#lL63WuQ0>RQ9zz9Bm)$q$B6be12x
z*)0N%u<OjaCuwf4hFQN1;$c|V+_@e%XpzxWwAOhk1=h!&Wa%)#fYpXk7jU9d5FUyr
z)pop|zsW7WT0U8i0SP6)0`uVI%HhV~%Z3etB0(w|KcMIU%v}Mmi?KpC`v;y^$nJzx
z^a#<%B2HT0)c4^7H-<&b&mgzR^;x!ZZJWO<ET>XE4}j9E(2Dg&DaC@-BniT-Ys`Ce
zAu%tb`Hc;@6dV&f&q7^wJ7DwD^G8-bSomZz!*!wAb8egCYnpKsFN$EKAGUus3nGry
zqHqWzB*{vIv_O3lzECiJ?v6>1o{sO2%I=b|9!JENkB)CAl^c$&YD6B!yDF+9XPo^0
zM3=?TJ>k66#2@+i8urQ0ExvxrRK73Bpm1qN!ZQ%)ES@$T`$xHx^4diB?C)!eHEp{l
zD_;7Wy_cF6u3dP&I?KXnG%{Asg=ruN#y3?`h;{Wb<;1wlkE_~X;xMH_34{8$VI4~E
z&Mv<h-h4{YQ+kwtWA=UvWSk>YT3X~IofB}f!*}OT;4z3-wgy6!bIii_fimTTy19pb
zNJ>qoD6iY8!!PMG3xc^R2{yjUDX<M#NzAo(3NKn3SA;oR33zLloBtr#JA(4)T1|R#
zFbj<OdvTrAo|i);l+=8`Ht9;>7ETw|c=YJ6rBod+1HLhcPhzgJv9-RDzFkD*=|@B&
zFAxVhiNXwOsOjO#t=+K(wyJS5Zci(yeC6Q4<Q8T>%-F)rXsBN#DAGdd^|k%_xpGcs
z;ZX;ZDng$@=$ni>;^gfawi^W@8>Mi6Al=p583`h5{6=3K9$OzNe0WsR(((-JYDBl$
z9&ksYlx)-A;40KvHCrA0&te15zfs5(U@LGl1EFs<`ANm7>B`DtuGsMQ;apb?(K18a
z?asOr*+>>Ng>Y!%d*WiQ<S@xn;blchjE95&xCTyJq?sFaOvs4TKKNhK1J$Ic|Hunm
zDdXs;hG;qeYzn33sCz)RU@oo<Gs^OsKh9vLIYc=?!^d2FQej|>K;#&Q@+k9tOVe$}
zZ^|3iAOo_*7O<g?*$i6pr32JDkSOywMg0E^)X*GO%L4sI+sH#7rAsk@`e=S(1jMN`
zpdUhWoJe6o=5Q1x>kr_Ds?0oKu$=j^E)|g5jxWXb#Jb#T`G51NowvU;bYLVkC<V(o
zIr6j?3==(u9&$y|PZ`X`{)tiGgP_I_cnA%@xc`M*K*`nra`)bb%^feL9APM?ffA?t
z{?!qR0GWs%WJlF$(M;`v$b8B1-^k}JwCz5b0`rGHCp`XK4q6Kxa(R$zQHQ{e8rKpa
z`2gI`akpax#5)Hj?}vdn=9wO-2e^Q0{D$Yh*hI=m1UuLp*E9?OBcKf`Aglp5nIfF5
zX9Rx=)xXz(fUOVS`f=(?rWulK$m7vA-eF4ywg1J8)4$On%gc7-gu`;j{F(C%xDS``
zOlCgL$&lZiMtY=kT=GkcHz;u5fPe!2`hPZu@ULNWVk*2)yfX7RW3N{NyABPUfp`D{
zsHb|Bkl}Zj5(o|TX*MUQdN~i}qlt;eF82;ZCAELaErZ~&aD}ea>|hz3R#ojy5NJ|h
z&7MZ!2{zbyAH=wlyzhwBzz1Y!+JPRowyHVYTU_*H(nYcoL=*Q@m`uDD`eJAT#kffM
z(GVlZE1f(m2Q^B{bmrBM0ePsO?ykV@BMe3Zk*ZAOAR#7^q22f~PJH0u_}VBEnibmV
z{0C;j2N5J+(2<tT4110^s}BLm%kiu~$Y~4+1ChNUF03a@!6m)}6GAJY?tm3Es*^_Z
z1RH2O9nZx?4~KsQa?(Op_uhxcvr@0xe^pKf*>T>vM~7;3Z-2eR9IT}PVcqd~uQTYo
zpwrdV2GwmDP29S~p|Puy{m<U-18uJ;wb0kw?fxK~69hTe%$rQpm&sPGfU5YYtqfdI
z|IL%x8Q%$@wXk?6`fv6$pqqtq=Auo|TrhbF=o2NNMAfI*BmzpxC6H5G@hgaK-vR4n
zHb}!+C4E$n48f$VwxjW}<j~P4<p<+EoGL%GLd4^5yok!Sd6OLU;(lNRpTPYrq!;(+
zpC5k+U81tuY5ZB#_?K1dYAx)g7fQc*an9ANaI|EJT)RE1w@lcE#o&v#4F?PdDz?gM
z7meo(tF=?53hjqVMC|(WrDmf9aVITtGHcT>jGFgAA+^khyuIRRIhJoAA^%Q>5A;i}
z5gzcxwY(5F=;kA_@)w6$+y8jO#b>cfF)a&$-;HD06|^{c?&s+jIk2n1BCJ~D#hin<
zP(2^+O;@=uzWnW$W;XE6T4!-p%(A&cu&|;*J4104;2e2tyf#ZpStMRoM6pOz71bRM
zFNChD*i06nl>BR{94Q!2H&$Vg>%+BSxn7$K$)EUdJ#UJ*%r_?Iv1GGpSmBgKwGPJH
zbq`-zW};6AbRgfsFHZ#rip-Sc{leetWGL1g%<$9hJpKHrsUeWAjq>;T%K{ZZn>_++
zin=`AEQVz}(=2IH<yPo2)9PW~yfsu}<r;X7w!DOHj)DI{=}Ly*-e)$=(`PSyt>$bD
zq50-3`@l*Z%%P9F2ET-No@$aX9sG*mEANf8rf>|oz%l~zW?L>g`@nNdVnNa*u*rEw
zGKoq#2jl2zdDCxMU@6x*d%D05EERAW|EnBMuxBIQIl<HU`E*;@^|e$5?{dLsUo#g+
ztI?w=xJ3;EO7VB}P#SIgq7u12yTy6sAPmQ{a_v-%7k#5iwi*?+p!w=*4o^jcFZU%r
zF49sowV;T_4!yIpxWkAH+I>-PozHayF>Mh>&DWf48BooFD6A2C5S1<+(*>9|W3{d@
z1Zf2AXjYxMX3pMA6T1a^-*iLzf1Y#$^rTH;(~@d`?`(z*USyL?LcCPqUI)!a=gWe6
zrK^;f0N1&WvU7DnO)WtE{?o(vA@y)KOl^?i1rEg#3ej=jG_f&didZvsZ+gwZvos|x
z)t)O3=m{zsXb_`Gj#jdkMzJWFhE=AsTov<J2{tIigEc}sz|s4knb;TCX4*3Jita{V
zG2nPyC@Md~&uxC`5`IHz>YelZ43fBOaqy3sso-kHFdIi>=Hi%0(U>BY;PaKAALklg
zLD@7&%dlg_9J;U)xZt|)ji0WfY?q(?cmX{;&0${DGzCd3>`WHjt@9_(Q$B7GH7vKc
z&F=HIwcQ^svl|7@04ov>5nDO)AWfV*?7>itH#+$Kn{BY~nwwg|{!u1HWo|8E5QkuD
zRsrp*#-&ze1?J9%VCrhnfac^zRo+=?l5jlD<%5m17CE&FlX!zr<kwDlv6&4Ib8k3d
zb8e<YX|eIoL;D%a2Oloo3g5B18bjUuw)p!x=6QX<k1gXT5P5*He6mTR=ni%;!!kCK
z*AZLJ&x?{hyI-Ag4RyQ_Bk|C8IVo%1!I~)R`jR0XK}9;x{fWfdktW*{M1{tY&UU0o
zwy~EtevA71ZfCjOKnI0*K(H_JIbB!ub(QN(Hy-`EMT*kk%jda{8uFj-%winBUkT==
zlsD;k5{LUt0F$*4yhr7Z#THFVTw<aqDgpu&i+HAzU0%zr-%FD~ui7$==?;f|;$H!U
zz=2^VBCUm*8+cMt@=OXrV|0WE^B0SMyjPDc-;i@(7#yXVN)Crx?}sBVhQN11PtfB(
z-cKk2UyaofyobdrK_}_AUek9piqZXr4}$B4fOoCsMac*4sa>JlZVynfA<!kHI&I<6
z5JdIvu4QFSj6Tz3fHkdHqblOVbON7ofXvJ8dpygR9|o!KffFw}$cEE}ba&0^*-iVv
zuji-U51kX07`sDp*}3&;o}Q#A?^H5zU)Apd1^YdVOoO`=o(sJ#TU;D_|44i7cohpf
zPGw%yOcaqkIj1l(sYu^-PXoq-sGKnW$t=%KH`4(+F7?dRBIR`DLGBPtf$>zSRU0FL
z>EzOMJbi<G8)+i?oMAW`N69kvTJ1vd1C3DWn3JtDI00xYO%}3^Lnvy}u>~1%ijZ2l
z?E~2ju4pyVx7Nl8#m!EK8mvr*`(8P%Qd?h=U0|8K`Af6%0tND+NYU46uDS^iKc%tV
z#h$<6^#ih5UJox44%mFt{{h$M{XUi|vc5utM7w4lX3-o$%@(DzA~x&2y6GZxZu^?(
zk;B|Xx&8F|I%9|Ce}B<^F6xTV1O_d9_gWFYn2V%mdq6F!Zo~v1@^rc>JnS*Ku_{$-
zTJPSqcIjSRJmMX6^;+*rw5ANg*2A#hj-S!zHo!WSiPeC$9e-GL{2}F3GRNP49$~LJ
zERUBYghdAEcau6Z{5dM1Q16ixM(!E(YbcIJ=8u*Ei7ANg21myt#zKbi;08%ZEzQYJ
z9s0x1v8%F5R{$Xq`X3c6_*ee~tzz6HAt=P<Qj5}nQA3|m()0X3|7%z(T+Z=za7?=T
z-5PuVE=6Xr+OvoN-uUA`r%xMioZGhnznh*PtOR$kpO@=E|I7y;Mq$Gjr=<VAOd*33
z?8SX>Xh7G;efg{7xFQ;yuSpxZ;7XkX5VI4Bf9{%`{n6n8xWF_~-aoIfrU@T(Thyiu
zj+c6`)g}DYt$EE&>os8gJw9_6#KY&lO#u0X6E2p+Q$2=N>{(2%^VF2`fBcBEiY3@8
zXTV5xs%<b$vfA%puhO<Br;<oO8yO<V@f4dYJ-M;oz+f{n*%*q~U?TDS$8VQWej;Vl
z`Q7TroBB~8lwSs=2Uq0UFXx%o1XmCt6o)+G7*__%hc+izp4afoVgT~1Vg;+PLRu(}
z_Ed)H{JA!4<XEk*w|=z;+;eHLxM%CKb?ARAjA?A-960W(z&2p`Udn!O;{H3K-yr(t
zqMmy`+L|Cp#EX^N4^4qH<^|d7t!6Ig7muYubK7$lS;JE9%22Y^I*iKM@pMiu4iur`
z2Wucsc!lijs`en39K60I`p;{k#@u>thF?T)%Rs_5Bly{_z9O@6ti%k@r$%2+3-$lJ
zQBzH>;0Lu}-(r|vZZ8e>JpH6wPaLW283ibX+31c$(G0;yAHuFG`C-C!7luIyl&IwI
zr=NNIq&_%trNKta@OknUb_eYCqd`;*#-ypH(f$G>?aoIR3HHOKEkS$0nl$qT1VWoC
ze~p9My9jt-BKn2Kg+9A$QYR?g_QLl13XBFW3mf(-(vK*3Z8#Z}yl@7cf#_;LFEc4H
zEHjSd)6Ulp`T#2R)O=r#S`{B?|Ho@3`v8OR`}I~m|1^E_tyM5e)u}yI8XYoS*%TL<
z$n@5v+}?oTxH?hKXHVi?&zOhi4~LuEPkeu!`w@V~9xsFXl~by;qZGZOUB3{+V%Rib
z7h$#2<@&Z%mmq);-v>~la?md@ltg<F4c8xh^8uo?l%0?3^pnyfPJw;Sc$9*}+OHN7
zuwUzm!bT6s+nzvD^Vx2=^z)21<+)uF$1>>xY9|hhl~7`Dj<CFL6Lm*r(I1=|6a%5r
zGW#}*e`2WgntL!D6*n_pXv_$^0`j2+huYnjiSnKf>)g}kckM4RD)VDN&8<*3OJ(*g
zySzt!(|w+#T}L;LSIQ_J^_bF9(;BbzcOe7JcyZb_`p3}-htaAsZ=j#hdch=OJa_<#
zgN*F|NLWAcM$pqr{!Eu2ppZQX^1M2^4*;Kp?>ty}&t?0m6gUg#LWc_o@8E3Nspb0#
z2dTpv8ZcK_yuj1ekJ_pI(JT^t3tlma6JE=AA;_D<NQLv;&HWRVf%RJ>laxi)-pi){
zMi%UGh!_S@)cS1uqMY0O^DzSz4rrGMh{NAc5U2$HuzoestV=--bRNq56W}ZIxZNXz
zYtbJ3N80x3mnY#Lps*Md1=e>J)i8KZtHC>+e47tz(HeiF(Cv3OU;cGzqjG!3h1U}~
zJ@ka3Z`Pg?)*Yo@)5zwwpPsE4@Z^5BX|dG8K}aEnmxE`SO|#r`<^xF41_#Q-dvScx
zAh`5=anKxn)nxW(GF*X4tE=B1^Z|1V@bNc#SQf(@w-?D<#H$9P$4AuCh}M+s1E0ZW
zkk;C;g;6#|Mn0dBCv{Mo$uVBNW%H)#CJ(AIIz~DXMwvGOXTx6W5i4!XrLU^LD*7XI
z23QH}DP)~nG3>OCH&3PAq^G#ab@f;5w)}jPnlS?z^QuoEk7J7E^#c?lnG~`9j&7k{
z?ra1d!wPaMa9@AR@f8mjDazRyi}-4SP{3)cEV9IAI-h*k=_T^6;eT914D^wqQu}=8
zyMazqjJhY26sAuh*yi>Z?S!)?JAF{*9ZsH1`Y#9;5;s0ctg$;xzwmKx%*?%|(mE?0
zjdQ3e@Q{-9od>^~>(OGx0@!1vXa1o@6X^%sV1Ehf{iI>O`F-t$_d=4G<y}Z4mONJl
zjsixVXlu4<bG2br#1FF}+~;uYzZ+gjyfsJ;0utZpcXQpM&#J$5rRB6?nCV#9!%7z`
zZ=*AlR}tx?i#$#wHJ(pY_0MX&%4S*uUxNWlJy)b1>}}oHfA8O*4oEqvDg@#87%!Gs
zGy(2Tht<s=8Pe7#P`0fbW78>;cEf&(T0wX)*}7pW=SCv+DV2v|)6J>vs*#jF>>^Rt
ztJsYH*fe&9I57b+)J5mUbvd$?r%5Awx88+6oTZ`i8>*X=HB7-L!mmpV>aEef)2>{?
z9meB5SAX{g`UH?CUN?UuqG{*aw*1|XasVso_}zuhEFVVfBX+_Lv4L=|r-3)&-Jr`O
zq#B4B$U(J{J8%C42}174;P{)P2c9J;3V4)w4@T|G{y%*PwuuwRAHVqAh+sdC8z}sH
zIs#K*NjJbj8x~JHrGFeED4n3V#cuQG@RcblGOiTo`#%vQr18`p`E?%5jr*^yjN?b{
zQaU>~RQ_amz@YL$3yOI(=~_aMRy@&mD<{$BWJUC{|90gj|L*uGd6eU5cQnWi!NH}r
z5W3F9j{|#Ub-KLg@ICi|LX)i(tNejo?XdW0?7Er8EBl_N)-3;w{mCW8iQ1&?y5;b$
zt0wR37mW{IFB&&0FOCHANLAD-Ik?)PiLRj!BId{6FZpM2(<al+o#k%=uXQ!ZR$JPW
zC2;K^;y+kzq}`JBTsB6#%-lyij@P~*r(e@oR0$h=i#QYwy~ySQj-_y?omF$Y=ZI=o
zuKpvf*aRTZoGz6+cW=3hAE2TtRs}>yzv2|!9`TNEw#eyQcRChMb_y4ESsN{V8=Dv(
zG07<O)DF#ejq}?}gmZLEm+q@f|5!7V^;4F)8@G-X!8TFwV}Xmfy&WVY!baskALQ(#
zV%g+h&Lx4cq&k?IsS1>Tx;k8o({3kI$FM6bJV=wFZ{bfu8P6p=Ti_0o3C6D1KaOh#
zwZl13qf-S_Za4<#28%66LDFx*Fkaao3|l=`-1ZNV+zeIT;6TgVYLf^OXfoLRO~z||
ziSmogAL{P^%~ig&H00X6*Zbq>fb6;5<LGVtEo*U9;j|R$ub)N?9}e%)o8%r9EvFs<
zJLv7u^)(*=;(u!i9ZN1Cs&h+~f|pQNtd`F)v`))Z0lf(Pl19w*ld#A$qG{GYlg!tB
zESgscj){XS7qURX;^j8j)~mr^!`FaA+(X1%VVw+Zy3@_kcx0{m-_yc6B3vSBuhge@
zCvJ3or-)bZ_J9y86<}&a`(=SAKS1Qwwz)WfzjAd1WyM!C;kD3Ka>P68d2tNe^<KRi
zhFlx`wZJdiJtR^goH4ofj0weT;0GkDBS0WK3COb|i6SO)`V~&*P)KHkYdSHr=XI7G
zXJz)_tI*m72M)NBk}cUCp>{Wh_4jZ&{Ob0}x80ok7Wmr0<(A@9W8xbp2bqlz&)7@)
zB$>Uc{%)rB9q)8^AePrWw9NcH{^e<26)<cz_kDhh8wFbIEw&1osRqFcc&R0AfaU!v
z4)kjAJXLJj_!g*$ID<K%|D<aSHOJZ_jJU8FtJU9M-$tCBD|%*su(!$B|Mt!E8EITg
zxyD=WLXi3{gK}Ak>=clXxB!1iL_3Yr(f&7KhKo~kvlEV^c$UeT30TKAs6Y%>(q^Fs
zWw|QGG7=kH-_kLi-e>1U=|s$(rMXh!K^z`}i(P4IWMh-}a08LB`{UdP(TGf283snS
zQt)Y*&7=n_>J>X3h`z#&zCM3Fqb$eGs3R4S*3zOQ1Jh{mSf9hA4{lfz%Hi+oic&c^
z`-Axnx_#0km_!}Ms21>Y!$G?Ij%TBY2hR%(zBQjs#(&gIG_2Kn6R+Sn220g2P)`oK
zu5_=^u&j0X06^L94%b-XE<)!=W%iv&!)k+<M#E{td`}xhbBacOFCV^0w`1P8Njo&P
zOtwf}yy;Nw{TvI}->Q_8I}+)1F|_`J3m5q(d<!3ajPM3DKw`O&Z!2jWicfDm=0n?b
zWrs26n6X@K{=1mOUpn!uz_e!AfqG@MdLSlurY(^m)?Beedal6lpw$0Mxzfjlb*kM0
zx!7M&&W001t`atI#uYRw#y?jHztGLB%y}g5?RDE7wFQ4mnY%zyD~ARVxa*UfMgBlQ
zDx#foi*c{myzy)zkNU+n+)<${ZfE>+21AAX{G7Aj`u^5XqHmJm-6&@Bb(*;<@a7NJ
z{*<>@9V>0e)8Cf;Nt<Sw<k&FB$Ms0wH1!_rNFT<t_e_2IQzK9`ys#aMY$I-tRJvB5
zI&(gW9|`J=QdCpdu<LCFl4>sf`3CK$e^LZy(nMJE2?Z^-OOG$P%>K09jlc1@nVmOT
z3Z7D68Sp;t^Huh9m{ai_h4MEkO<`m&KXAvEfoDQ8_ROvVP*3m$43-;Lx_H>J$KC~t
zupm9G6g;9a43W~j97k@!Y8(Io1DodiQBZetZw#gNgfg!TpSAF6V1_1g2W~kF{u7R(
zyNnU|eDC(UM>J~OK~dQF^yL1|4$+}6MrovLnt!DHKz(#k!ZV#OlccC5;}b~EFd@IM
zZSIJ)-b=RHyHK185{qq6`M)DFze?$v{o;bmSr*5bYt0E_cHJMsq4K)PJ0SxtpY4Ud
zwtiRjkB>gy=ZhFZb<(X(3_jPo5g4i1ybc=i5;`(C`rX9q4;!Go;=x3+gmrl}TQ8|-
z-}k(8Rv-dPZd;N1Z`HkLSW|2FHz<M<4gv|Nfb_0MI!Fz@D@8>GrGtVfRRn@`3<A<b
zlp+#P5$u2>HS|!F5>ylfq=YUt^zN+9IsbW{cdl!`%*=;(&L>ZlknFwhd#ztvH}A0G
zCvL7CH5qz-%d%(D1<O9S2Rv%RXhg%(uUFz4kDRzRQhsxiRhh->t23{FF+LdSjjIFO
z&j-P{+|(4as-X2gFr-}b*#Xzgb)VWCqhvRNeduTV1PamcQcH@2JHfun!Q+b$E5%WV
zvTlEqdLT%w@y)rHXMfngTf#KpkKJphyW>R?QVrR{E$-*PNgbqo(epwMt5SFwKy5v*
zws(JG9UPkBn9OG}hdP)--4eh>+d0==PB)tUKvq|M>!DQ#)X~-#13q7v!lYqxQ0C?E
z`7~*Vh>wSotlwWY$|&M=D|Xncds;4im|;kD#-Sk3EX!lYIdowbkS~KQNqw{u;`yRG
z%J(M3hz~l`Pd~Kp);c*}W5ml-_T{_Uu{)L09@3XzXOv9-i5S-@*&aNzsJV$|<`o=~
z;3j^SyfmS2iKX5C0R*!@#QO~-+hv_wuR4WU@?voX5<VGeR2{mmHy6K2wsXuO47X&v
zQZeqg2GXua==P7OUhUxDR1a9VoT?sttCxOQ@TFtJU4?ht8Ze1d<BfP@FmP`v`B~z1
zGiZt<-sg}2*JHjl8<JDBsbg9_!7Ahr6w<dgs6d001=x_R)=?#g2uc|MpjmLM$49&T
z@Ojb}5EYO<urdAfgS(yNQ)P`LJ{Bd}yv2J$<81Axl#Facw|z70s{ZT*s&>ibIZ0e9
zDu{I@kO#RB&9cEtWF^zHv6HPMg^XXC0h@0Pe4pF<`udaBGHmQ6U(hE=1k7ix5EL)5
zbGk)X0r+J$auJi%9>=Kdvb*Dc+?l6rQQr7-@kdUGv+#=QM88i=Jj2L#nB4i$ok7Uq
z>0{X|cLNT-V3{lmr)QK2y`B|=p@|HmHKhXy<vBw>0JpNQ&rfBDrTfXf6w0M~IAJI5
zL=o8bT%p}7Wlet29!^XDs~-BFTelSrNIt{KZofj8I@9N?{ifF#7~znL;A<5U?`mt3
zDeIw*>||~om_-q1bDCNiyt(quq5e*ahEq!^hA%!fhT7yaGPl1V2%<Y_YYj>}v&2Qy
z7IGfjSgm)fcWgmpMWg_s>>fpFNR|Df`kSa^VSl3_4)anTZR@zTb!M8+bE3S%CCwqI
zPJ6;ul~=QuDZj2DcGX-|Y^VS?X6k5(UCbDLdIZzHPjOs1Wb^Svx)m|%k?MoT)oDVF
zahy&*ShA+{xZG;0gw+o<=(aAj!vem(w`brG<7y20B~Q%wZH8DEQH)2ZkZ79l{NKY2
zHOFI9^Dn%;gwO|L=g00omS*^~Bp-q6d~wFFCAQmH_b&=3&3hcxkxd-8H%)lbSX4q(
zizMmo6KzA1;=PQG9ASS~A_n9xb74K*$$-wpDC2qTuxoZOj=-QuWfNQ0#F|SkJE%Eu
z7acOwg*N9+Rn;#*-1{=@F>BHCBssA?8Xi<`j$<**&lJkbZ!2tb-uFJsZ)Uq4I3G0e
zV{59QL30uPWEH!}dv!-s<N|fNL8ipvV&dAUWp;J2N{K1F<uwi61Ubnf4%No+Sf2jJ
zwbU9*uLEAj1rpki8D+7pcv|lyybBlO-Er_*wvbduljxfy1v6u1Vq64FYabi*Nk=>w
zFv98^-zbXOpPs7xlX6TnbKs-dvjZCBI}LIK7i~ErZY(DrH<ln6;5F2ufO}Y0|Fb$Y
zG()JHk7^v!Dx;)}Y8d8j3Nk|pj^BrvTia*g3%#V;Xeo|*(1B%9IKAU!1niA3p<<zf
z{GP&K#@`q_qlP_Q`&Lm~x%7Gbo+}Hr^XJ&wYeq)u6;PAe_30fz)6b8|1`Gw{Ze9ES
z>ZgR<XCa9~qrZ<5WcJ6zBw8(WM#gYDn3YDdll$)G?M%0X3FXd93w3tyPZV|F!X1y!
z)H}USe_@X&iW>apnKhAhWOQf{6?p2b3Nd$fo73vouWiLG&SJRKCEPn2qS-P2qWEXS
zbj0>hb!kig4$l>Xzqi-~LPvSp*{ML1NcDDqDdAwZf^<@Pq=kK^itmjR*E*x)ST?=g
zyxhJHM7!{%+{TG^MoTkE5ip-@D*Qf5vm`K4xXYAAEJsGT+7{z*V@f&VaLHMtl10g-
zu=AX6=Nw`?*7BI%`>B^L#XpW?EO{B1M*U0ZZoLc{@3`iJX79}L8Na{&Xh7Nr#hi&~
zO5DODO+lL7`<b~`L_AZBo*9qPYHNgTW5_%zH#@NWl9z-DECg1vb-a)%&OWjC3d)Z7
z7JE>OzL3eg6;kFM2dQ<q8yHfMLFs(WKABgjgWAfd;dzVSIM;MkS#YB+o;@i_Can)O
zFsKj+G|O9^^fx%Pl+K+4hf99*EircW>Kw79(>6|9n?LLm$o!>+WE_mCBb_l<V2M4z
ztt3rpv)_9;oiXzF_t#cjxEJ?&{YFOBMvaScIaK*?9@rlbqR>dxM?&<l*<TL)(I*_A
zv)yWz5vi+qH2$Z&xBpjP&CbD5Ga8d<{bBO<sz#*Tyush~W83-L5B-(Q5`0L1iwRNw
zDr=_9h{W-U8A`Tth<VK1zzV!q{vwd8d~CsqG}X|+(exI!D6_Z{b~35kM~hVWS&x_f
zY|yv|W7D1LdpgoS?BT;Bz|u)On5k9b#@5V!JDNtvGKKK<sf`FvMqWSL=y|~N1X)Bk
zV_;pvCt+yyQ>2uSvd`cB_H!os)3Rnhj*Yigrd?Z<Oi`7?%mR!1pAzO7$J_~7wML60
z$!3>Oe2esp&5)3DWDzmvYY(svWBXgbzwhEGjnAy5;~RsV>%3_8<4_OGVhl*A{)?K{
zH!6;hrF@?hhNyZ33W!rXWy=ORM@~M*nbV0!JY;~yB9n{a6g67Wc|05-szE1XF^jr7
zICD7X%S)80sGhb&UCH9Jp$3evL!Af<F`+YRh5uIB@||q4Oe?RUsPz{@$r2F^0&Vs_
z*IB~jqF9p>^O8TYdu|js)C=}s5w(J&9g4$C9=Qjk34b(s$DLZ4>sHuB-nZ|Db|+`)
zmydi&^BD@wiHgZ<PKj&>*`-K1K5@7?wI(`$#b~XuZk@p!#xfq(1%m=(b%{G;X~UGr
zw40&+;is7zf5bX|riA~P{Z-MsPtd7B$%iY|pYUM-h~#UYiBEzS>>URk6NZIP*c@a?
z;$Bnq;D72QLlAQaF`2M#x)t(gi@A(LU)sl3fG#$8U1HImZ^%MPX|3&stS8WrZJ5?t
zBFm#Pnnao9#c{Ea&1f&viLI0}Ll>)+;aRqE?)HYQahfS5G0JI5t>M?`VCPvw_r(fy
zp&)h3F3l3nV=Sod*TtAP=w~dS<HHzdCdwizga6C%;da^m@nVk5k=~)j)%AN)lWyZB
z%U7s8EW#yi6N#4izXIro$GV%k>gVRZ)%Tv2)Xs`|82mzx?U;5#;e3R8-|XN{XL|6b
zt`ea>|Ccm$^C_8|zu2BBriZC7m-M;ZbuM+O*}r-Ffe6=1sm>qMH5@Zv;1XBUxzayd
zM(v4O^Q9;Nap`k4-S7Ftv`ESW!IugohvsU7ZntPho3Ak%&texeCH;e+a*iJ&NjZdX
zF|(<5B{f>N?32|_Pd>3&=s%&UJ$`7+NyC$V(NS%A$tT=war{7arADaCz-hA;CIUHP
zF&53v#Y(0*`i;%>l$$D^nT4K`<oxxi)BVpvk5lb(oyy!36*sK&r|Sd6=&n#=a*cWA
z>yyJ>_L{jAY<X4G({+gznOqC&9)BygZ|Svpu_cOLrh{~1@i24$WNTpdy3j*YCxx;<
zC9a9vWha#qrHz)|iM=9vH%0{TN<5~~k7<>{1rw;+K1p}^M@gUgQqZron$MlBQ%3cQ
z+^maFF>tJ&%lZ2BZEUe^bS!5GjOZRmWokC>>&N?49?AP<{EqZ-=V9_;TbMI|@n_r#
z?i`vS9otz{b@J_K+q{yhrrT_#)D!x#R&DFM{_5wFj9;tAcSil@RK3n#%__@B3ocW}
z-5l@7nCt5f$BX9Nb}`)2Sld4DQ-Ky-Pr{jsZq2-(-}pcyz;)_{{tX>2bSs)%W7|59
z9tz#ruGkL$Yc71hw(7isLEwiwXbL4vSst6HZzYB|4ETSdj{Qe#bD?Q+!Qxku(|>e7
zgH#o*G}JvZcLRteg78-9T9XV9qRi?3)}&m_;ZTB8YzEuuI~~5|nye%`6w3OH86z|l
z&o|`X{H;k@ub$7p^=0RK;vGU_^~>BhAPbcbHzig8sc-4h_{cPmPd!NKo%&OrrUaf3
zTg|DX^9t-oisZWWcQ%4ReenLc86+LJ;)XaJ{3)au;o5F`8VQyn5bTl`l5pw_OzBGA
zPcy)3;wt!X?t^pN4X36AlM9709Cm>7r+PlV1`M;=2XgiJk9<2m%v=sROGD$D3Qzm0
zfX*FDjRx60BPBN{&+cM%U~MY8>_UHSaMJMhuV_00apwUF-b$uI5RA}!2d)%RYbV#$
z^*uCX?>Z`H;!wC~+;BejT_o~pwhXaZbeI5eH<bb9ke;eoS_L!&Vy{y0l$d(KE_k@E
z-%+JLPoz^Ox(4C<A1nJ@N(Z+a9zlOF2F_+YMas@&bZrSfvFd$W;B!tj`D}r)qypOf
z-WzA15)|Ti)q1Xcy59i$!Ka1S7v4T<NfL_UQoeCnSM!_X!qeHH{hL`=dUOKoXPhg|
zR^3?=Z5+2*CA&PU{pV~gszZX=GW*RFR#zb7T}1m}`{@kVsZ8=dDZ)DM>!I)z++nz7
zXX%k*z|aawk0v!3AONR|<`Hhe*chmjgIA>q9eS~cFVYXu=%op~n79Xgi>^w7)slnO
z_&&QBRuMsMo^F=#9r(_dlpA8%{lOxU9W2K4<^1!LhfX!+nq|%27;m4s@W#?xpHu6_
z+#Mz^>vz^=yBZiloeuA36TJLAWwt?NeaO||!H4N4`|q#L{~*A5stwG0=JttY7GC<`
z9IvJsRT4B*7x}|rrzSgu+kRRRIGEmc#)~WCs-p-ex(!y1q#a=?NpyecpJ0T|=uHWs
zR1BobcBiUrR@tj<ZmO6oeqNcmY`4M~RY$tJD=>R|3sHll6{vAd?1g<V&Inl)*lYRE
zeC#R*2ItW|M?SpBUXC-dU8T-#k366oaY4ze95A~9Xjr1HNnU6PBv0o_V1@kP)C$ec
zn^jGJ6EFi=p*v+Q_XKXM6^uERd`eZ{0jBmBqG(PH;D(uA`#;f+Y(4&~GH}(fzmr~S
z9ypPag>mxS0%%HL##3dBL<~w*S39!9mkmI}v4LCO>Jw-A+<L`{Zy_IajfW;YgAkGU
zxG?bzFhd!y+AYiK^KUPGNBktBTb~k(p%LoXV5Tws)twUjQZ68w{mN`_30YF}g?ZVH
z(Kqdg6dCYtQiIMP*6)nH>6<@KL5T(?-n!2a%NJ*Yb&6Ib9r3iv9wG^|X~Y%i9Q^Ul
zrdMSV21fckJZ|Cl)~X}{ABaQ00!hth^6PWCGwU!-t`s=9bbjHsrWMf#rkRe1$ZoMH
zeJ?@h+qWD1fA5EsQ+&n3=i|SkF+$VfSVy9qbL&K7BA@3K!MmSm$<p*qaCYk?^x0Q}
zwQayv=A`{N2qI3`Qx)7Vt-^fKOY0B-%Lu6a$&%=sE_<a9Uh>#{@4m4#`3s?hM-dW3
z^u2P&gn_Qq)QjURsw;B?0}sV7eocsHU=y{>Yyxbyd7s{wfDL%hlNL<{pICPT&C`#d
zzbWWj$KBI#kR_EhZA!oR&toxfMg{VMB};$+y!&J{j#ts+aez0b=q5ViAl1llkK^Gv
ziGCBjP~lh7F`--anRx#P3Bq&0;glJt%U;CZ@IsHZXI`I-*3%Vm+=8A%7Q>l{8xzo4
z$knJ1{wzBaj0l+{g5s4g8oy2J7@*0!tOwouwkEz%H$je}m~l?B6g|_7;N{zXiAPR+
zcJC|wa-}7uZAH)PtCV$ihx2a}zF!dHPLUdY@3H**dy>D9P#9*R^GmGD%JY6$rA298
ztx*nWogJW{E>^x^TVC?rfpe6NjfFyf?zj!~{MSv#ipWw=vYre#T%5vO1|w^9jLVO2
zme?mK6Vmaa1Gvb^doWV-Go54vGp0!$n1yIS2Uu73hzDCoQ?F3)@O0l|WT5sB+N*eu
z9lh`-t@0Rc?0KwtDE>NJ@Hf1*Xx1vBo9iR-ydYi3B~7jl*J|S>e4n^sTRWphLBZ}M
z$vOUvI*TJK1ROqgmc)BjSb=*^T7YATVZOi-0H!gKl9Wrvq*i%eRa|Esm~N08-QOU0
zMc%Eu1G^bwSFDxR1=Lv8@Wn>1nW?@!Gd?!*^eJU3cc~T#I3O{_aslwp!?Qo31BqMb
zGUrupmQ{fjv^|Nkupy7<rjk)g+Q%m}DZUH>HQk534V(8kwxsHWi~u3xsC&rv>>HgZ
z<}rcX8Ta1e6IwjRp)gCs27HR{JQOA^BB<zWMD@o)+;(lZ_Q!QQ&`I5q7Ge3pc1w%L
zE^y|A1qk-1bd(lj2Em2!K|+WC;>@FL+?Zg*3Va6v{v}2SN%LIem?cI6?z6C+_Q`0Y
zG5ZDHP;F)0Q8d{neD6MEy!k*jF8498C>KM|h^;)IiFWF5)T+RcI|Ux$R_HxSs~y2B
z0@b{<6v`v+@3^|^UqfKV{@B4tVkS88h6&;K(sY6mknKKL4{n@F5tkv04>6%sNs3SQ
z;{6KdW=aM;Y`<r_q$$w2j_4T6^#25Z$<Nm#2+q(hfJHJEvoZ;XrzSl|xJnWSKyu~o
z#8R8$XV(4F7+&;ev8@O%Fb=MhBy5S@wvka5>}Jxpn^o$LLm<@X`gIU1-eCI^YIjg)
z>g0%M5$*^t)(s}DmihoKs<8&TE52Y{JFI}^w^3!li~I8&8*#;FB!{>joLWPTvJ$j4
zsNlI9foK1sZD&@9R?XRxHy1xVsu|A~R>&k?W!w}bmYo`$GVb2bjw(vs0v6&^b-2)l
z|J?aM5hbkQw-uA4e#}o(&WnoI2R>m4))W>JjhWK3UQA)4<KR87&4W(J89*!|4op$!
zJ6y4+4V!899!@+#xWtQH$|v>~xh7Y-3#FN2PfT|18;Z8SQW>?uf7ys7#h{X!vgjqW
z{zXCen6F;z8NQwb|5DhPFNErf^4e#y3vzk4`$Y%V3E-0V&4?BwoL*j}JHIie#qHFZ
z9*FTm+YxEG!ERG6&$C{#<qr?n;e#gWu}!#-lP(LC%Fbw4yh_W1@bO2)m#Qp`XI=V>
zYkN&`=&xPr1#ES?&FtYOTmeqaKh>q1p*sQn74*}Wv@zp*q7_IOmU0?x>525E2$m<&
zVM1*K8)9v-S5di<4r-HlQGQ`e;K)?(<8$Rv<@*^@+1@GAw$#V(wDHu|F3#i_FWl`C
z%0SnKFsyQ}sjr6G@Q8PXuKtnbXq-D$L3t_X`}7Tt*_LIKsxw&I==Dd8lp<kejLn+S
zxeFZyzSQd;j#;Zr{s8MM+9|TAL>#*pgL~GwTg0_ukq24%f!dSf+K%g@DJciRa{ncp
zn>e|}F|XOQH*`<^uRGPR5|H;$3iVP?#T^!8Zr6X)R8fi8wImTi7Wo4{8Z>7_DMkr6
zLc0ntz$)iR+HGgRpQZo5_dnBnUt4>RwJBbu$%h}gUlfW+!`*S%W6GdI)>R5xD>(l*
z@t0j~*<JmWJSIe%KpN#@(zn7;_FBKivg=)v{$Ti#D?Y`{F^71r_`X<h2MxKi3S!kA
z3W=eALSAo{#qG|gkYqlagUcQKlq^;O^}Dwc+Rn!Lp?XAMr2s#fC|9g8(BS}2p{i4`
zct;8%Gi%-6UnXkusFa=;SHr}m&`RnX@RdOyulmJJ7X(r9UYoi8N|UXq5;b;oJ1EM}
z0zsCvex@Ja2V-c4q2czNs_)eo!m0<ZOnZaMP&C)+3m+DAvTGioM|tyfAos6y%`l0=
zS$n&q_2KbU#9DN{yWq)jTViQTvREQGrB_^^-n}dMqF(#KmA~h{1?z0x=>A}&GIiUv
z6NA19o0x3AchL;nZ(%7FNOkfe64@%$wsJUXta-FG{o5~yYj}XNm-PPW!>rR*5$?-V
z>p6CYEUWI0zVRuIJGkrcQKk^Gd~rXbp&<bW7+YKwyQHy<%}}8i$VaLYQf@caC-buW
zkS0&>?m=%bz$Z+zi??(ooBh`NgII4H>D4!<<+{p!ulJ@nO-AvBc!A~pE-vesTw(dT
zd(g!PJZ@P7iSBj561!dple>q=@mz|2VBql<Gkq+b6k%8n!3E@G7Wc_%vwfi(lVXZa
zjR#)Kx7f~0+Ew*^zWDLlLx$s3ANWRva)FoUz?h7fi52p=g75$y{4i@U$+v*1ZK}Vr
zVpOj?Vu084*d|!?h7Zt;^-^KBYk86DxAMtvc^N1Xc?Xc4CE>oz!sMa?^uC2>AtQQ(
z2Yf`$Z1>(0PemD{ZD{qlFkv|JPb_9r09M@gOdi+pQukaKt-C!FF|*=|IKpMGD%x5z
zqjl9IAiWw|tUj9TPAt{k`XDv0X6>>Oss60J?*uBBcK!4P=yi?d<$f%&_`W*#^8O~Y
zQ$pK1s-L11Zsn_OzK@#^fz$Bb+P!Fkc5c?AbZEb7_Dfn{+YinrIo1iBA#_Vz0dQaP
zlght%#!*3a7mqJ@>n^DMJtuYg>bKT(xf#oaCTraEE)<hBqQdiJq$gdcc=EwfhhF{#
zD1d5gIKXBQi=OH2A~8M8Km8=)_4(H+!Iau6$c!5|zt?bF1J%$MsfgZpI2<bwT@n$#
zZ9=z~J!&KN9rg&f-z5w~iGvT~(XTBFpSENu=FgZ{-tsMhj`tfexrl{o1M<`=d^s!C
zpd-Ep#)+JAjXmYy>H$}v^5PqFbz47O)!rmBaDG?WnLw;nd}0(hj~7y)z8@oJ4G>AF
zM))$jZi3(KJ1E_S+_q_hEVq8*&#xR4@2|Azf0i)Mjr+(|7ywSaau{FG+GFs7VOca(
zoBt}~uwmr{?Lwm!5diH}XpafkJZb>OJRnMY3c2&8`P;rh%Mb2fp6c1aB7gg5o*SG}
z&7HTmj>c}i6mrj&G8`6s2IjR3W<B#T<gHsKEX_1kGILZlbSLPwobTw<x>SSEFH-yd
zI#{3}&AU?<u1dEg)N~$npgu&K1#7|era?GR)V_&%=Gm`ka4ioFcjqHK0x8m9?J*D5
zKhKgHha#EZ?Mm84S3&P6O}%3w4Hw*FYmF*;D>}xRlNz<AeNZrM{s<*W*IMAwyzG6x
zy=CCQN@3r-jyP3e)5-8XJSvzQlc`54<a@HOigpeh5D2(^%|&Pjv%;gA9P5GoDlA&@
z(p)MxU&Jiga|IjH@EHH>q&ahHpkoCrs0P^P=nFj&t88bi^KI4GrlZG^V99f%<VIbg
zUek2h2E35tJ=sqgXpR*fH^B`C9<7`kT0<?7U^cMqt#sk}aw<r{-I;jz(DAYT2RQ84
zzm9?3-O_IFz)LpDL9#?0TcO#s?6z(e<`re`$XZCYIHmTboMVd26Q<6trVHjj0MmPJ
z*YeOduEdBD3JRZ@ME^eXO#h-<Wl5HwY)H`uyXp`a5G$#G{x#B&7_P2u?9?gZ)EMW{
zQ{-Aa1Iug{CVa3?$>C4<A5c0hO(b_74;ng1;?pVGwNHm?JWf^Ch$8<a96U|{_vb8<
z*<iCry!UAibQ~0Rt<Gl#EQnu(?GnxbHlB$aBh3zitY;!EDd9SNe}`+{0yrY*Nxa(i
zF7u(54bkGkGD?S7HPRr)6V|~idpUY0mZNDiM$?H;_prm>&dAz%9{q7Zg6#`{KX2-y
z_HRa*>Z6Chi!R}B?G>PXkSNBU6pw3cNwCaiMp<sqA#i5y*bsyI(w;(c1Km^R!d0;E
z>;1lqx@<@M_2ASC)RnXBy6;##vn6nrU|n(zhV;$0Cf_}&C%jedc>F!S^{;-U4}6S+
zsuT{bBfM~GLrowMH12&I*GUo%g4S!7+a7(d{Ij#E@S`Xk=UnV$$L!pr;ZR5STFm!J
zrFGD0<}36Ka(mNaBX+ho4SwtQ<6}BPqlb`814+i|Q>piv8XI&^!#-|Dg3PF84$&(x
z!s|c>^Tv6kw?HoM7g>hPYV$WZ{Tj5jbgb!~JsdBLbhXYoY8oH!|9*f|Htj0O5CH~l
z-{RPdfekt=7$$b;OLVpX9Zl_>Z0)dx+~{{U75cV;21oS}2H{P3@0L7a=BoTtlt6R+
z4e>x~9MU}NR<07yO4bn}zAzTK!lS>5eulreH2rh2sDSC~9u^6QeGd{(uL_7A5T_iR
zQ}Gqi-II89z?EwNQ{xtEk<o#EmmNfuTz<}n9>ATMWVf@ZLr<vo=yR%fM%<!KHt_qd
zQho-;V#v|U2&t3FdCJa3cM;eGclyIE*!fjpiFAL*h3R31Y;QY}{iSYIiV-U4qdg_P
zjvf!*SQ52=S?x(X991*^v^Lho?gx^GqGW76i*aeBGdt1iQ>%M)KPA0z?1|p2p#8Mj
zuL)Hc42=tb(?Z___nn{Ue3?9&(Y?zxL_g4hN{E-_ClCeBqSdCSDkE_;KYs)!V=8lm
zCeY-aa5*W$HtntH+w{d3KRAo#md+#|U0MO40_WZxezd`VBqpPwIm^-lo5*oR$QnZ&
z^5-{~bP$ccxU#r6X&KZguV3Z9a!Dr*863N~Y;O{KM~tk{<Of;zaJhV?Oc+R?SV?!N
ztM_z0oS6`-<1t(u{ThljH2A4Cq=DqPaDh}TZr+~l%@P^>XhOcZ8t7!a>uDB}h_9hB
z)+ek`&3_Mcyw1zdJf*QUC$9e}3eg!6S_$Stx8ZV*MRmJ0CYG<!Gsnb>P2bt6eH!Bu
zdS<Xe&Vg5a@e#K1_LfY@L1dtduqc<A>=kLrB_`?*tTboBC_=~Seim-H>*9EqdO*nU
z<$(wmvS<VwO+&9dx-i;an+0{9KgAckZlWIZjSHrZQiOm#3Ajt3`Hi7&#zj>T!y0ZI
zb<<Mf8)@rew<qNnZY_KgYwKNyTYOQ8ZVugOWu$g{M$?{pT2O!krA5czfBAQu;zR$7
z-%=8IR4-XIdxHCU%vZ!XYzuH&YYpQW>^$o7SImb_MbMb}%#<BjrQ~=o!;)jc%gt+Y
zub-Bj(WetscV*+X`7FS<AR>CiOgbQ$g<3Vz;^&q~pGsVBrw#0k31)~Nj39NTSM0$q
zu^%J{u{MUYXED<8W{>O<o{s2X(v{f$R_tyyMf=+@3-kH*YAO*!dJt)FMCycNnu_4H
zNx6kT5UCXpl_E@D?*aVJ?uZKPG2BvMLA_^k(!XdeFYVlSF&iP@0*=RS&)4{f-)Vwb
zwvaYSK6s3Pd9mfNUz;<$UPV`GZ+h=4&IEHAw}rMv9q<1gmY00eJOD2<#5<iBttTzm
zNE}~%XkqJ}sp?-Gggzf6=D|5HOZl=CFjO{LtR5iP*iG0dl-SGWZ7^+I6xDb%)^yZC
znWUzjDE${+eEI2Zs`cj99*4XBsaq3b+UDq#9RVF(;khc&OnT9P4sw+HX;GHybq;%C
z5!QNq>SQlY`u_Xdqk?G6x1x-otBRxiyi~$w&`Sd)JO0@W-%bW<Tv|iy%v7AE$U!mg
z#nu&UX;TCJQ$9?B*e{JeMk@5;b$un_jC&JkvJ%)isi(x{Td-Tr4H$Pyc5ZtdPhI)~
zC*wv|`Ip1Ie~~tMf|vw&x7C9;x)a08y=88D7uM`U1^7&V`NP_dJb1qa^x%X%cN~}0
z*&rN#wN-HMPX*i*mKf^TGE!R=eH>rN{xPI?{)SiJV4(aEcRwFl<aTYbdb`qNyP^fQ
z*MUkB#Dt(`a-hv^-{sMblGF2QFa4r6<AR@kS1xrgHFbEw5-_A@aPULID5(ei&P5tV
z`e|PgJEyoXT5IAsu`aW=s>aqIAvR!#E7b6%2BSR^5rB7A@@fR!!(oB4g}-9o<V5=i
zs>l^G$17Ij48`Y2U++__SD19vLoNR7x#AOEze$XV6=gocBG-QlA6ziKc##FchFig2
z31-WnYmC$dK3~V6pS4%3G;Zo3Z?F4V4_^Ne#;199ND2NQZy5alPD?@F1O0!KTRL`d
zW4B&w3-dWt9%3p^#+ims_k3)b=`KVjjiI%LX(2?ZuH&hvLs8zRJs@sV-YEgmqm5I$
zfBRU?*1X}Vh6ur&-xj-QvzzZqH}>-=2FUk6aU`F={*4+z0&N}a{SW^Q0zslHNR;~c
z^S|zF$VsZhZy){FL=KVHEF)~zK8R~asotFE;GA+j>N>Pc)efXgz9AW&#PTWMAOWc;
zPE@%VFPp%p;W4m&c{kAnzeeN{$>(v$y+Ws^asZmWtAiV`WtV|3C$o6Rt^0#ZVx^8W
zamNkbrNq5=H*W#&@)jzb2gLkVwh_<-#NY?@lcuVGeR=G6Be<U~kAFkLk4YEuOnKPw
zxYM_%AhfM#0hl6sHT~AmlNG{mFSW`%l{CMli8E~DfU1g_oie<5C<4c*h+(V?uH^jh
z%OANsN^6nnw=2QmsX@Xf7xG7n5R)y$dZH|L<81KlU+;!h$^iUAmG%P(B-JkD=P!tP
zi$=d>V;a-z0glcaETf5UoC4_<!CwG~EqHvaEyHDMA_p3wa>&b+f~Bz(!p`0A*?+nW
ze=$9YO+9YWzC6>b_?`%A3|DEH&v;MuIKD0K8!s~<+Z{T5M7v-<D!G@43}l%F3axMs
zR|X-pS-;u7$-|P?6Nja2-GbTWv2zb-f>@4h7u(nFJ+-(zoE@|=nP-xdeE41JZvIa8
z;*E5;3AkO(+KOI~weoZjqzg>2ADEb&m7$?}2M$sM`n?0KED5`xnR^1UeOkOacPv%P
zPJQw6H|9EZNJ)==%{7@qn%rE|%-;a!=YAtN8BqU=!XGi1L>z##_~U>^TCy~%kuY<N
z_5O|wvuX&&w+335-*g3|s^BRuaLmm_g|z{c2Wam~s^C6I250n#ZOC_dZjK}P94=mx
zbOJmA7qh&mdvJ{am0Vq|4Le_*Z`oJ`%;yhiyKn)veIr04-_v6;|0D?BFFGLEi+Oi2
zP{tI&oqCkYv1VK7sOOJBBQSEOO|*gb5_#-&^<4Fz{wwW|f8u$0!q*VDw>x^?6q&3g
z${moMfR17nIhp2vxX@TkQ1jFAetOnn3>h3X(r01~{J~fL4veAiK##aU-*XPI9vc-2
zqq$!yy6Uhz7G94yVL-OBNMYgi?YV2g1J<S(HnDPf*RQ1z<Lh2|HUm#V5zaeqVZsjE
zjw@R?Eqh?5hFM;sX>9luMngNj36N><@AI#2K5DZ1XBCT=nLreWh|}L<E!TQ1F+a!?
zapXyqKrY}_=e<dHgzjQ8u;K=#^t}T2q7HCU5ZQ35JAsVQ6vX^!gCDhEp`-lPG!v(A
z3dHg^2A(<^sko8qi>m?+H$-10SnH3FTj9s+T?iFW0fc?eXj?`X(-HaRA5m&#T&`<|
zBJsQRx%qp-gj>vpo_OEEn1N#Vj0}0#`z;TSB?DWM!DC<R#e(;J`1g6imj79y^{_%o
zQSL>a1N|y!sTZqw=uE!VDHJHU_oT72e`yVgrnG0OR>0)4h98K4fw<*G?_;7+UUYvu
zay>CJr0usDaHZa~y=Ms$0MAf{gq6bLkwEZY4j8w7B_r1GNWwg)LeC0_7-f*j>(!%S
zX8=ogAtS9g_o8w0zU75RJvs^}S=Dz92BQnPaw)hK_;%9N`@kx8XWX{02D@y}1_a+b
zHu@O@{`)CnZK1v04gJ51V3NExgP~Z?Gth2-V4_d|5J<^0$r$^ZXXfb{R|7K;;;(p3
z5PdKmFc+at7u8lHgbmqRGdv>m<Bd+Ixxzp&kBnnBRUY(w{fsD-Oph*3Po6ACNw=J}
z4<^zRWam9XVeD{{Jh7XE_sRC2FJ!ti{wS7h<Dd=}S$prZ{#n64*QH~|m9*wXmtU!n
zhG7l1s@^q7?;b$3*ed{Tc%6l?PatzTR+*c;u{`BMxKuT)%%glGrvipq^#OTxeP%dZ
z8T$j(JCYF4xHw~XKFZ~>in;pGA$%?txV|I}C4qMk8R_#NMPn|o^P+QXa4%xKXMs4X
zeEtmB$HQH%I(UvD@ga-v`=;t?!_@tdVJSuQ`!JIz_s@eSM>v)*TS4%?Cfa+2#e*4!
z0NC7`e4fHW?fLJ=fsT4NZR*RjCRgp0Ib0m31?P?VhBB(-m@m5U&iW_z)tfvWbSE^-
z@=&6$@fI&;>WT@S#AIS|h8(oK7qk!Gx57dx`${LF-%nyQm&C&H^Lg9p43kL8;JH9K
z=9{|R?@$=a^0QZ_A3WT#jCoZY4&20wBg4yqIr|nz+qiLQG8<zs>n*l3Tz0}NPWmvN
z`jtJwBX$;1Vn>EdDwm3x{&@_>A<|6VdxZ6^%p1n4*q{EagwtOm^3#V^T@UQpq|=k>
zecvkPKK#w!v9vV@YL|f=wynRHyN&hKsds6^IluXLBXq_q{%Mf!(R5rsB`%??#Pv_=
z7Jfmj2P^p{_rJ-g%kM|zV#d!U{2L*Yy|@6o_1{ggF6#eeC85|kZ+U0JYj>)KqzVEo
ze3VRfD?W+dv0GKN-qk@EYW@<_pT4GD##lSrCrYCtRxk7<Ht+NVAJRFXl7jHd7j`qw
zyFVGtaqs^VI$SwY`%jL3-ONFxuPME!nQw^A(?qIMF_=SM0XT~4tq%Re>3(SU?@!-@
zj;2F#C$t-w%)E2;z}ihe)pSB;7ezB3iWxr`%}x%=6?K|cUXx$>=O@7S5{{#`u6ADj
zC)ZutfxPl>54NR-{JMYX{2j=DnDhUe{tjhfCI1;kQqu;ByuMK3WZ;Q(i$CDW&T_MT
z{VK$hk?xK(aGSJf1i|Bxjn%oiT^qdr+^j7$5{fq`POrkO5KD+pTR!1|(x~jTUB()V
zgK`oH4{+4=@2?(ybO*9K?|D9iE!02_o`sru12Rp8HsW;gek`y8)L<y;JK(DPH0k1J
z=z$p8)Bp#C2k@+U&$HtHkz**@5CalM)xx5!;8wm-3Y%kTu1N-BGAQ58jJ{gc{AORg
zuWEC9l8%&X<`Vcj6v+x9B?W9VS!c$|?mfUc&m%sHNAj+Ydp8=nJa2+c!UsB>;w9MB
zFWmytxsS_}I0SUz7^1no7K4<uOxIIAMkL-0>Zl@Hlbal{r-qyWZ2VwT^vkc>c=b?c
zVAnaZ^Z_yB!cDTC01p6GgaGN!!JZONPr-zOc-TqHfI+L+oU0y4!88HA<d-aF=7(td
zR@EVuk+(W%-q;awg}v{u#6Hi5tPIlBjDAXPQv0*D=<(^k$ZSnGZC=duKdo6GQ04wG
zW>hz2eYNU!epOE}?DoZRrzJ1G&-uO&$z>x(0U=5%jdQ@x)_M~5ssYeg)0;47#u{dd
z^nruGEq&*h#k<GX!K-t5^MCo|Tjp4y9dyJSRc{J@>ig@|ipkszH=aLX7SNh3Hd2!A
zK%_QfLPyt`O!$R(CJsT4I2f`cDR$7O*z^+vOu~}IC`)jVC?Ybb<ayo8A^GH)W*V;r
zdbfrN5M0_GU1=$rg!U!gjjXq4W*u4;_jE6C8oakuli7%<dNSLMq{e|dp2;t&MB2`G
zFtr3TxBN5hp9fO%COBkwjuB#L8qr~RC4Nml+#rqBI~cBl5WsIg7}i%<<r;G?9z|H>
z?P0z@UckG=kd!_!J>i2PD|hiX#6cB!C}r1QY*27#3Uk~j#^txJq5jLW&s%B1w+jBL
zw>$?pw9~!rXA1;@0R6Ff0T!uyHAG5baLP4oyCpo*Z#^TlIfvRJ_l4#M7sGsl-Q|h6
z)ALA3<OCVcG7dlBOeKTNEk`}&r0Cu=g0zS5+&WWh+{}*d_pa*nUz+$b7W1pn$w0V2
zoqnkqoGU*GH@12eDU(H4g+&O}h@YhbG^Cq|Uq8kUud)4E9jIh4D70$se8e&A(U>HS
z!U2(!54?~V7bJ;_ih)WaeT6WDd+ZbDBLV{WmtNADZ~to{4IZNcMd4(TV~An=j?60g
z1y9#ZtqKV&hy@2!_9V*0ixmmDv_C#v9xjI5AI1FF?9I6BdI~3HvH%b44cjX%8<=_Y
zyjkTli4o@0n)S*K1kgq)*2#89m0Y)vzD#T<c?CY>!Qbf6AF$mJNHeuOwf2^W`oG*k
z&f;1t5$Ti|pWGyQbfT;P1L~S5{un<&5vCtN=+0F;J%RDA3#oJ<>V~0^sS?37R^O45
zPS1ncutf#dFCMT2KatH$&qpCwYwjGgtbSm|n^PF5M?nIMx#o$*riq#yDS-qvqsCCU
z7Bk9|IF@)i$>;yFN(0_<&*`qP9N1sm+xPLh1>?6Gr<UWrsBeQj&A8l)lILEW*2R8c
z$pRhgqvE+%NNK7Z`0`1XN*X^tmacTU|42l^xLPZs;Z+_RhCtr?8!|s^AOXM9!B2ZK
z)1!LZ?=>3&I8skszYx1hP!BHWI&tkmz@wN@=MGo7Ise>X;(5H~90b=dTDFw^_vImM
zBp>XzYE_MW7IW!$oemD~gFa;}F6~*wj}H$OLDC`(#>30YJlgaEqMj8q3X4nc6D+!6
zOKY+9^;aJ#*6N=lE+!yy%XjuCr?_u+%p0>gYgsGKee4I*mFpz#i7T4`FY!)-#z}^q
zFMs7%KeL%8D**emo~4@g{J)-->b3omU4buHc(2(>Rnb?rej!OuMa=m+tC9j7M@iae
zDau!IF>JDUjQ|m6qsm(e+n>agA)<^_P<9o78rqdRv*is!1oQj<c~j+m{|+UguJThK
z@BLSGUJFD@9~3f|YqI~@OY<}*xcb{#Yp93sb=FZDVnufKYi}(f_`xTh7xHc0`^Vw-
zJU_y*3EJ!RT`kMIR*?Ew`$_0op&`HBQx&WYvn~ElMVsdT7s9vjBV7N->#0yH`Z?=<
z#*PEh`{x3J87Hv-w-UiIC#$V2PM{ik_7=ARU2FKSif?VL%SH~Gg@GN&ilCE^NH?>r
z@O#VPT&(IzN&hQJrUmI5pimvnK7&9TFu?4+1v?s;V|+c<7e)`i3UB(Em#Ii4y8V<r
zevk%{T@f*=5vs4rhy8km?4R&1YnBsU8vz(HGW+CnHXMz;+lh2&zGh9pm~e~fy}#>%
zs%ZS5mr;!}O*-#m|MeS4-b8meIB+xm+lEk{!dT^h8?OKT{0AfX;49<Xh?#o38CEsB
Q2mUkCGu3^kZ6EP}0JE0RNB{r;


From 5e852c2fc424d31b19d98fd2cdee6ad1e1fbee1a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 6 Nov 2023 14:19:20 -0500
Subject: [PATCH 687/922] update release note #9590

---
 doc/release-notes/9590-intellij-redeploy.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/9590-intellij-redeploy.md b/doc/release-notes/9590-intellij-redeploy.md
index 4c6ab29ea9f..07af352ece4 100644
--- a/doc/release-notes/9590-intellij-redeploy.md
+++ b/doc/release-notes/9590-intellij-redeploy.md
@@ -1,3 +1,3 @@
 Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
 
-For details, see https://dataverse-guide--10088.org.readthedocs.build/en/10088/container/dev-usage.html
+For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools

From 02a570aceb89afa0b9f110f76137959d5704b2e4 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 8 Nov 2023 11:34:53 -0500
Subject: [PATCH 688/922] #9464 Add commands, endpoints, IT, etc

---
 .../harvard/iq/dataverse/DataversePage.java   |   2 -
 .../iq/dataverse/DataverseServiceBean.java    | 124 ++++++++++--------
 .../harvard/iq/dataverse/api/Dataverses.java  |  35 +++++
 .../command/impl/GetDatasetSchemaCommand.java |  44 +++++++
 .../impl/ValidateDatasetJsonCommand.java      |  45 +++++++
 .../harvard/iq/dataverse/api/DatasetsIT.java  |  41 ++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  21 +++
 7 files changed, 258 insertions(+), 54 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
index 12b7e41b3d8..943a74327d5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java
@@ -597,8 +597,6 @@ public void updateOptionsRadio(Long mdbId, Long dsftId) {
 
 
     public String save() {
-                 String test = dataverseService.getCollectionDatasetSchema(dataverse.getId());
-                 System.out.print(test);
         List<DataverseFieldTypeInputLevel> listDFTIL = new ArrayList<>();
         if (editMode != null && ( editMode.equals(EditMode.INFO) || editMode.equals(EditMode.CREATE))) {
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 5942d4a8010..407cfb343c4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -44,6 +44,11 @@
 import jakarta.persistence.TypedQuery;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.everit.json.schema.Schema;
+import org.everit.json.schema.ValidationException;
+import org.everit.json.schema.loader.SchemaLoader;
+import org.json.JSONObject;
+import org.json.JSONTokener;
 
 /**
  *
@@ -924,12 +929,12 @@ public List<Object[]> getDatasetTitlesWithinDataverse(Long dataverseId) {
     }
 
         
-    public  String getCollectionDatasetSchema(Long dataverseId) {
+    public  String getCollectionDatasetSchema(String dataverseAlias) {
        
         List<MetadataBlock> selectedBlocks = new ArrayList<>();
         List<DatasetFieldType> requiredDSFT = new ArrayList<>();
 
-        Dataverse testDV = this.find(dataverseId);
+        Dataverse testDV = this.findByAlias(dataverseAlias);
 
         while (!testDV.isMetadataBlockRoot()) {
             if (testDV.getOwner() == null) {
@@ -997,7 +1002,60 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
 
         return retval;
     
-    } 
+    }    
+    
+    private String getCustomMDBSchema (MetadataBlock mdb, List<DatasetFieldType> requiredDSFT){
+        String retval = "";
+        boolean mdbHasReqField = false;
+        int numReq = 0;
+        List<DatasetFieldType> requiredThisMDB = new ArrayList<>();
+        
+        for (DatasetFieldType dsft : requiredDSFT ){
+
+            if(dsft.getMetadataBlock().equals(mdb)){
+                numReq++;
+                mdbHasReqField = true;
+                requiredThisMDB.add(dsft);
+            }
+        }
+        if (mdbHasReqField){
+        retval  += startOfMDB.replace("blockName", mdb.getName());
+        
+        retval += minItemsTemplate.replace("numMinItems", Integer.toString(requiredThisMDB.size()));
+        int count = 0;
+        for (DatasetFieldType dsft:requiredThisMDB ){
+            count++;
+            String reqValImp = reqValTemplate.replace("reqFieldTypeName", dsft.getName());
+            if (count < requiredThisMDB.size()){
+                retval += reqValImp + "\n";
+            } else {
+               reqValImp = StringUtils.substring(reqValImp, 0, reqValImp.length() - 1);
+               retval += reqValImp+ "\n";
+               retval += endOfReqVal;
+            }            
+        }
+        
+        }
+
+        return retval;
+    }
+    
+    public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
+        JSONObject rawSchema = new JSONObject(new JSONTokener(getCollectionDatasetSchema(dataverseAlias)));
+        
+        try {               
+            Schema schema = SchemaLoader.load(rawSchema);
+            schema.validate(new JSONObject(jsonInput)); // throws a ValidationException if this object is invalid
+        } catch (ValidationException vx) {
+            logger.info("Dataset schema error : " + vx); //without classLoader is blows up in actual deployment
+            return "Dataset schema error : " + vx.getErrorMessage();
+        } catch (Exception ex) {
+            logger.info("Dataset file error : " + ex.getLocalizedMessage());
+            return "Dataset file error : " + ex.getLocalizedMessage();
+        } 
+
+        return "The Dataset json provided is valid for this Dataverse Collection.";
+    }
     
     private  String datasetSchemaPreface = 
     "{\n" +
@@ -1053,9 +1111,17 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
     "            \"metadataBlocks\": {\n" + 
     "                \"type\": \"object\",\n" + 
     "               \"properties\": {\n" +
-    ""  ;  
-
-
+    ""  ;
+    
+    private String startOfMDB = "" +
+"                           \"blockName\": {\n" +
+"                            \"type\": \"object\",\n" +
+"                            \"properties\": {\n" +
+"                                \"fields\": {\n" +
+"                                    \"type\": \"array\",\n" +
+"                                    \"items\": {\n" +
+"                                        \"$ref\": \"#/$defs/field\"\n" +
+"                                    },";
     
     private String reqValTemplate = "                                        {\n" +
 "                                            \"contains\": {\n" +
@@ -1085,51 +1151,5 @@ public  String getCollectionDatasetSchema(Long dataverseId) {
 "    \"required\": [\"datasetVersion\"]\n" +
 "}\n";
     
-    private String startOfMDB = "" +
-"                           \"blockName\": {\n" +
-"                            \"type\": \"object\",\n" +
-"                            \"properties\": {\n" +
-"                                \"fields\": {\n" +
-"                                    \"type\": \"array\",\n" +
-"                                    \"items\": {\n" +
-"                                        \"$ref\": \"#/$defs/field\"\n" +
-"                                    },";
-    
-    
-    private String getCustomMDBSchema (MetadataBlock mdb, List<DatasetFieldType> requiredDSFT){
-        String retval = "";
-        boolean mdbHasReqField = false;
-        int numReq = 0;
-        List<DatasetFieldType> requiredThisMDB = new ArrayList<>();
-        
-        for (DatasetFieldType dsft : requiredDSFT ){
-
-            if(dsft.getMetadataBlock().equals(mdb)){
-                numReq++;
-                mdbHasReqField = true;
-                requiredThisMDB.add(dsft);
-            }
-        }
-        if (mdbHasReqField){
-        retval  += startOfMDB.replace("blockName", mdb.getName());
-        
-        retval += minItemsTemplate.replace("numMinItems", Integer.toString(requiredThisMDB.size()));
-        int count = 0;
-        for (DatasetFieldType dsft:requiredThisMDB ){
-            count++;
-            String reqValImp = reqValTemplate.replace("reqFieldTypeName", dsft.getName());
-            if (count < requiredThisMDB.size()){
-                retval += reqValImp + "\n";
-            } else {
-               reqValImp = StringUtils.substring(reqValImp, 0, reqValImp.length() - 1);
-               retval += reqValImp+ "\n";
-               retval += endOfReqVal;
-            }            
-        }
-        
-        }
-
-        return retval;
-    }    
             
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index d0711aefa5f..fabb33e328a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -44,6 +44,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseLinkingDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteExplicitGroupCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetSchemaCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetRootCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseStorageSizeCommand;
@@ -68,6 +69,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseMetadataBlocksCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateExplicitGroupCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetsCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.ValidateDatasetJsonCommand;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
@@ -232,6 +234,39 @@ public Response addDataverse(@Context ContainerRequestContext crc, String body,
 
         }
     }
+    
+    @POST
+    @AuthRequired
+    @Path("{identifier}/validateDatasetJson")
+    @Consumes("application/json")
+    public Response validateDatasetJson(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String idtf) {
+        User u = getRequestUser(crc);
+        try {
+            String validationMessage = execCommand(new ValidateDatasetJsonCommand(createDataverseRequest(u), findDataverseOrDie(idtf), body));
+            return ok(validationMessage);
+        } catch (WrappedResponse ex) {
+            Logger.getLogger(Dataverses.class.getName()).log(Level.SEVERE, null, ex);
+            return ex.getResponse();
+        }
+    }
+    
+    @GET
+    @AuthRequired
+    @Path("{identifier}/datasetSchema")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getDatasetSchema(@Context ContainerRequestContext crc, @PathParam("identifier") String idtf) {
+        User u = getRequestUser(crc);
+
+        try {
+            String datasetSchema = execCommand(new GetDatasetSchemaCommand(createDataverseRequest(u), findDataverseOrDie(idtf)));
+            return ok(datasetSchema);
+        } catch (WrappedResponse ex) {
+            Logger.getLogger(Dataverses.class.getName()).log(Level.SEVERE, null, ex);
+            return ex.getResponse();
+        }
+    }
+            
+    
 
     @POST
     @AuthRequired
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java
new file mode 100644
index 00000000000..48f135dba32
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java
@@ -0,0 +1,44 @@
+
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+
+
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author stephenkraffmiller
+ */
+@RequiredPermissions(Permission.AddDataset)
+public class GetDatasetSchemaCommand extends AbstractCommand<String> {
+    
+    private static final Logger logger = Logger.getLogger(GetDatasetSchemaCommand.class.getCanonicalName());
+    
+    private final Dataverse dataverse;
+    
+    public GetDatasetSchemaCommand(DataverseRequest aRequest, Dataverse target) {
+        super(aRequest, target);
+        dataverse = target;
+    }
+
+    @Override
+    public String execute(CommandContext ctxt) throws CommandException {
+        try {              
+            return ctxt.dataverses().getCollectionDatasetSchema(dataverse.getAlias());
+            
+        } catch (Exception ex) {
+            String error = "Exception caught in getting the schema for this collection. Error: " + ex;
+            throw new IllegalCommandException(error, this);
+        }       
+    }
+    
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
new file mode 100644
index 00000000000..a01aee1db9e
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
@@ -0,0 +1,45 @@
+
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+
+ 
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author stephenkraffmiller
+ */
+@RequiredPermissions(Permission.AddDataset)
+public class ValidateDatasetJsonCommand extends AbstractCommand<String> {
+    
+    private static final Logger logger = Logger.getLogger(GetDatasetSchemaCommand.class.getCanonicalName());
+    
+    private final Dataverse dataverse;
+    private final String datasetJson;
+    
+    public ValidateDatasetJsonCommand(DataverseRequest aRequest, Dataverse target, String datasetJsonIn) {
+        super(aRequest, target);
+        dataverse = target;
+        datasetJson = datasetJsonIn;
+    }
+
+    @Override
+    public String execute(CommandContext ctxt) throws CommandException {
+        try {
+            String retVal = ctxt.dataverses().isDatasetJsonValid(dataverse.getAlias(), datasetJson);
+            return retVal;
+        } catch (Exception ex) {
+            String error = "Exception caught in getting the schema for this collection. Error: " + ex;
+            return error;
+        }
+    }   
+}
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 56bf53c1c99..14131c0fa57 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -162,6 +162,47 @@ public static void afterClass() {
                 .statusCode(200);
          */
     }
+    
+    @Test
+    public void testCollectionSchema(){
+        
+        Response createUser = UtilIT.createRandomUser();
+        createUser.prettyPrint();
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+        
+        Response getCollectionSchemaResponse =  UtilIT.getCollectionSchema(dataverseAlias, apiToken);
+        getCollectionSchemaResponse.prettyPrint();
+        getCollectionSchemaResponse.then().assertThat()
+                .statusCode(200);
+        
+        String expectedJson = UtilIT.getDatasetJson("scripts/search/tests/data/dataset-finch1.json");
+        
+        Response validateDatasetJsonResponse = UtilIT.validateDatasetJson(dataverseAlias, expectedJson, apiToken);
+        validateDatasetJsonResponse.prettyPrint();
+        validateDatasetJsonResponse.then().assertThat()
+                .statusCode(200);
+        
+        
+        String pathToJsonFile = "scripts/search/tests/data/datasetMissingReqFields.json"; 
+        
+        String jsonIn = UtilIT.getDatasetJson(pathToJsonFile);
+        
+        Response validateBadDatasetJsonResponse = UtilIT.validateDatasetJson(dataverseAlias, jsonIn, apiToken);
+        validateBadDatasetJsonResponse.prettyPrint();
+        validateBadDatasetJsonResponse.then().assertThat()
+                .statusCode(200);
+
+        
+        validateBadDatasetJsonResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body(containsString("Dataset schema error"));
+        
+    }
 
     @Test
     public void testCreateDataset() {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e3a7fd0cfc3..0443bd2e35e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -406,6 +406,27 @@ static Response getGuestbookResponses(String dataverseAlias, Long guestbookId, S
         }
         return requestSpec.get("/api/dataverses/" + dataverseAlias + "/guestbookResponses/");
     }
+    
+    static Response getCollectionSchema (String dataverseAlias, String apiToken){
+        
+        Response getCollectionSchemaResponse = given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .contentType("application/json")
+                .get("/api/dataverses/" + dataverseAlias + "/datasetSchema");
+        return getCollectionSchemaResponse;
+        
+    }
+    
+    static Response validateDatasetJson (String dataverseAlias, String datasetJson, String apiToken){
+        
+        Response getValidateDatasetJsonResponse = given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(datasetJson)
+                .contentType("application/json")
+                .post("/api/dataverses/" + dataverseAlias + "/validateDatasetJson");
+        return getValidateDatasetJsonResponse;
+        
+    }
 
     static Response createRandomDatasetViaNativeApi(String dataverseAlias, String apiToken) {
         return createRandomDatasetViaNativeApi(dataverseAlias, apiToken, false);

From 521e8d24eaf1fb9e0ad144a336a06ed384f4a55b Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 8 Nov 2023 11:53:23 -0500
Subject: [PATCH 689/922] #9464 delete test dataverse

---
 src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 14131c0fa57..c43a0c251a2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -202,6 +202,10 @@ public void testCollectionSchema(){
                 .statusCode(OK.getStatusCode())
                 .body(containsString("Dataset schema error"));
         
+        Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
+        deleteDataverseResponse.prettyPrint();
+        assertEquals(200, deleteDataverseResponse.getStatusCode());
+        
     }
 
     @Test

From 7c630f70a4e548bf51c4ed27ee1a50825d795379 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 8 Nov 2023 14:13:14 -0500
Subject: [PATCH 690/922] #9464 add release note

---
 doc/release-notes/9464-json-validation.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/9464-json-validation.md

diff --git a/doc/release-notes/9464-json-validation.md b/doc/release-notes/9464-json-validation.md
new file mode 100644
index 00000000000..3703b420225
--- /dev/null
+++ b/doc/release-notes/9464-json-validation.md
@@ -0,0 +1,3 @@
+Functionality has been added to help validate dataset json prior to dataset creation. There are two new API endpoints in this release. The first takes in a Dataverse Collection alias and returns a custom schema based on the required fields of the collection.
+The second takes in a Dataverse collection alias and a dataset json file and does an automated validation of the json file against the custom schema for the collection. (Issue 9464 and 9465)
+

From 720b3b0f488482b04cd0b7f1c15d129a387bbb79 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 8 Nov 2023 15:00:28 -0500
Subject: [PATCH 691/922] add doc for get schema

---
 doc/sphinx-guides/source/api/native-api.rst | 22 +++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 1992390410c..e7e7c6fc280 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -505,6 +505,28 @@ The fully expanded example above (without environment variables) looks like this
 
 .. note:: Previous endpoints ``$SERVER/api/dataverses/$id/metadatablocks/:isRoot`` and ``POST https://$SERVER/api/dataverses/$id/metadatablocks/:isRoot?key=$apiKey`` are deprecated, but supported.
 
+.. _get-dataset-json-schema:
+
+Retrieve a JSON schema for a Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Retrieves a JSON schema customized for a given Dataverse collection in order to validate a Dataset JSON file prior to creating the dataset:
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=root
+
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID//datasetSchema"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root//datasetSchema"
+
+
 
 .. _create-dataset-command: 
 

From 7be534771370f0e588920c818cf0b0e42ead4111 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 8 Nov 2023 15:32:21 -0500
Subject: [PATCH 692/922] #9464 fix typo

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index e7e7c6fc280..5d784eebd64 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -524,7 +524,7 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root//datasetSchema"
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/datasetSchema"
 
 
 

From c553d1b3a1e7b69174477c48ada4952ec2bd9e3b Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 8 Nov 2023 16:15:50 -0500
Subject: [PATCH 693/922] Add permission note

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 5d784eebd64..f170cfd53f9 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -526,7 +526,7 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/datasetSchema"
 
-
+Note: you must have Add Dataset permission in the given Dataverse collection to invoke this endpoint.
 
 .. _create-dataset-command: 
 

From eb963307890e61f1c5124e7b367bde9803cc8af5 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 8 Nov 2023 16:35:06 -0500
Subject: [PATCH 694/922] refactor calls, add reference endpoints call, drop
 globus://

---
 .../harvard/iq/dataverse/api/Datasets.java    | 24 +++++++++-----
 .../dataaccess/GlobusAccessibleStore.java     | 31 ++++++++++++++++---
 .../dataverse/globus/GlobusServiceBean.java   | 13 ++------
 3 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index cbfe6e8c5f5..b1c528f3fd9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3573,7 +3573,13 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
 
         boolean managed = GlobusAccessibleStore.isDataverseManaged(storeId);
-        String transferEndpoint = GlobusAccessibleStore.getEndpointId(storeId);
+        String transferEndpoint = null;
+        JsonArray referenceEndpointsWithPaths = null;
+        if(managed) {
+            transferEndpoint = GlobusAccessibleStore.getTransferEndpointId(storeId);
+        } else {
+            referenceEndpointsWithPaths = GlobusAccessibleStore.getReferenceEndpointsWithPaths(storeId);
+        }
 
         JsonObjectBuilder queryParams = Json.createObjectBuilder();
         queryParams.add("queryParameters",
@@ -3587,17 +3593,21 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         substitutedParams.keySet().forEach((key) -> {
             params.add(key, substitutedParams.get(key));
         });
+        params.add("managed", Boolean.toString(managed));
         if(transferEndpoint!= null) {
-            params.add("managed", Boolean.toString(managed)).add("endpoint", transferEndpoint);
+            params.add("endpoint", transferEndpoint);
         } else {
-            //ToDO: Reference endpoints
+            params.add("referenceEndpointsWithPaths", referenceEndpointsWithPaths);
         }
 
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
-        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
-                .add(URLTokenUtil.HTTP_METHOD, "POST")
-                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusTransferPaths")
-                .add(URLTokenUtil.TIMEOUT, 300));
+        if (managed) {
+
+            allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
+                    .add(URLTokenUtil.HTTP_METHOD, "POST")
+                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusTransferPaths")
+                    .add(URLTokenUtil.TIMEOUT, 300));
+        }
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addGlobusFiles")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index ca029947462..1d98044b2b5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -1,5 +1,9 @@
 package edu.harvard.iq.dataverse.dataaccess;
 
+import jakarta.json.Json;
+import jakarta.json.JsonArray;
+import jakarta.json.JsonArrayBuilder;
+
 public interface GlobusAccessibleStore {
 
     static final String MANAGED = "managed";
@@ -11,14 +15,29 @@ public static boolean isDataverseManaged(String driverId) {
         return Boolean.parseBoolean(StorageIO.getConfigParamForDriver(driverId, MANAGED));
     }
     
-    public static String getEndpointId(String driverId) {
-        String baseUrl = StorageIO.getConfigParamForDriver(driverId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
-        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf(DataAccess.SEPARATOR) + 3);
+    public static String getTransferEndpointId(String driverId) {
+        String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
         int pathStart = endpointWithBasePath.indexOf("/");
         return pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
-        
     }
     
+    public static String getTransferPath(String driverId) {
+        String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
+        int pathStart = endpointWithBasePath.indexOf("/");
+        return pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "";
+
+    }
+
+    public static JsonArray getReferenceEndpointsWithPaths(String driverId) {
+        String[] endpoints = StorageIO.getConfigParamForDriver(driverId, GLOBUS_REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*");
+        JsonArrayBuilder builder = Json.createArrayBuilder();
+        for(int i=0;i<endpoints.length;i++) {
+            builder.add(endpoints[i]);
+        }
+        return builder.build();
+    }
+
+    
     public static boolean acceptsGlobusTransfers(String storeId) {
         if(StorageIO.getConfigParamForDriver(storeId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH) != null) {
             return true;
@@ -33,4 +52,8 @@ public static boolean allowsGlobusReferences(String storeId) {
         return false;
     }
     
+    public static String getGlobusToken(String storeId) {
+        return StorageIO.getConfigParamForDriver(storeId, GLOBUS_TOKEN);
+    }
+    
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 3a9f75b7846..dab0e36852c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -1351,14 +1351,7 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         String driverId = dataset.getEffectiveStorageDriverId();
         GlobusEndpoint endpoint = null;
         
-        //ToDo - consolidate with GlobusOverlayAccessIO.parsePath()
-        String baseUrl = System.getProperty("dataverse.files." + driverId + "." + GlobusAccessibleStore.GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
-
-        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf("://") + 3);
-        int pathStart = endpointWithBasePath.indexOf("/");
-        logger.info("endpointWithBasePath: " + endpointWithBasePath);
-        String directoryPath = (pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "");
-        logger.info("directoryPath: " + directoryPath);
+        String directoryPath = GlobusAccessibleStore.getTransferPath(driverId);
 
         if (GlobusAccessibleStore.isDataverseManaged(driverId) && (dataset!=null)) {
             directoryPath = directoryPath + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
@@ -1374,11 +1367,11 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         }
         logger.info("directoryPath finally: " + directoryPath);
         
-        String endpointId = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
+        String endpointId = GlobusAccessibleStore.getTransferPath(driverId);
         
         logger.info("endpointId: " + endpointId);
         
-        String globusToken = System.getProperty("dataverse.files." + driverId + "." + GlobusAccessibleStore.GLOBUS_TOKEN);
+        String globusToken = GlobusAccessibleStore.getGlobusToken(driverId);
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         String clientToken = accessToken.getOtherTokens().get(0).getAccessToken();

From a080f84b8cd20693e14a227993a6c6c33510c020 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 9 Nov 2023 09:35:12 -0500
Subject: [PATCH 695/922] #9464 add doc for validate json

---
 doc/sphinx-guides/source/api/native-api.rst | 25 ++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index f170cfd53f9..1af98b6c20e 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -518,7 +518,7 @@ Retrieves a JSON schema customized for a given Dataverse collection in order to
   export SERVER_URL=https://demo.dataverse.org
   export ID=root
 
-  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID//datasetSchema"
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/datasetSchema"
 
 The fully expanded example above (without environment variables) looks like this:
 
@@ -528,6 +528,29 @@ The fully expanded example above (without environment variables) looks like this
 
 Note: you must have Add Dataset permission in the given Dataverse collection to invoke this endpoint.
 
+.. _validate-dataset-json:
+
+Validate Dataset.json file for a Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Validates a Dataset json file customized for a given Dataverse collection prior to creating the dataset:
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export ID=root
+
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/validateDatasetJson" --upload-file dataset.json -H 'Content-type:application/json'
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/validateDatasetJson" --upload-file dataset.json -H 'Content-type:application/json'
+
+Note: you must have Add Dataset permission in the given Dataverse collection to invoke this endpoint.
+
 .. _create-dataset-command: 
 
 Create a Dataset in a Dataverse Collection

From 7d3836646c1fa72262acc8a1de6242277b5948d5 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 9 Nov 2023 14:23:53 -0500
Subject: [PATCH 696/922] #9464 add strings to bundle

---
 .../harvard/iq/dataverse/DataverseServiceBean.java  | 13 +++++++------
 src/main/java/propertyFiles/Bundle.properties       |  3 +++
 .../edu/harvard/iq/dataverse/api/DatasetsIT.java    |  2 +-
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 407cfb343c4..bbf35535915 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -18,6 +18,7 @@
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.File;
@@ -1047,14 +1048,14 @@ public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
             Schema schema = SchemaLoader.load(rawSchema);
             schema.validate(new JSONObject(jsonInput)); // throws a ValidationException if this object is invalid
         } catch (ValidationException vx) {
-            logger.info("Dataset schema error : " + vx); //without classLoader is blows up in actual deployment
-            return "Dataset schema error : " + vx.getErrorMessage();
-        } catch (Exception ex) {
-            logger.info("Dataset file error : " + ex.getLocalizedMessage());
-            return "Dataset file error : " + ex.getLocalizedMessage();
+            logger.info(BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " " + vx.getErrorMessage()); 
+            return BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " "  + vx.getErrorMessage();
+        } catch (Exception ex) {            
+            logger.info(BundleUtil.getStringFromBundle("dataverses.api.validate.json.exception") + ex.getLocalizedMessage());
+            return BundleUtil.getStringFromBundle("dataverses.api.validate.json.exception") + ex.getLocalizedMessage();
         } 
 
-        return "The Dataset json provided is valid for this Dataverse Collection.";
+        return BundleUtil.getStringFromBundle("dataverses.api.validate.json.succeeded");
     }
     
     private  String datasetSchemaPreface = 
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 79887f7e76c..150dd9048a1 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2670,6 +2670,9 @@ dataverses.api.move.dataverse.error.forceMove=Please use the parameter ?forceMov
 dataverses.api.create.dataset.error.mustIncludeVersion=Please provide initial version in the dataset json
 dataverses.api.create.dataset.error.superuserFiles=Only a superuser may add files via this api
 dataverses.api.create.dataset.error.mustIncludeAuthorName=Please provide author name in the dataset json
+dataverses.api.validate.json.succeeded=The Dataset JSON provided is valid for this Dataverse Collection.
+dataverses.api.validate.json.failed=The Dataset JSON provided failed validation with the following error: 
+dataverses.api.validate.json.exception=Validation failed with following exception:
 
 #Access.java
 access.api.allowRequests.failure.noDataset=Could not find Dataset with id: {0}
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index c43a0c251a2..24fbcf13d23 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -200,7 +200,7 @@ public void testCollectionSchema(){
         
         validateBadDatasetJsonResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
-                .body(containsString("Dataset schema error"));
+                .body(containsString("failed validation"));
         
         Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
         deleteDataverseResponse.prettyPrint();

From 7887a0527ddeceabd10a62bde68fcd38ed9cd824 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 9 Nov 2023 14:51:55 -0500
Subject: [PATCH 697/922] #9464 simplify commands

---
 .../engine/command/impl/GetDatasetSchemaCommand.java   | 10 ++--------
 .../command/impl/ValidateDatasetJsonCommand.java       | 10 +++-------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java
index 48f135dba32..2d5e1251614 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetDatasetSchemaCommand.java
@@ -31,14 +31,8 @@ public GetDatasetSchemaCommand(DataverseRequest aRequest, Dataverse target) {
     }
 
     @Override
-    public String execute(CommandContext ctxt) throws CommandException {
-        try {              
-            return ctxt.dataverses().getCollectionDatasetSchema(dataverse.getAlias());
-            
-        } catch (Exception ex) {
-            String error = "Exception caught in getting the schema for this collection. Error: " + ex;
-            throw new IllegalCommandException(error, this);
-        }       
+    public String execute(CommandContext ctxt) throws CommandException {            
+            return ctxt.dataverses().getCollectionDatasetSchema(dataverse.getAlias());                   
     }
     
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
index a01aee1db9e..ae1a89c3661 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
@@ -34,12 +34,8 @@ public ValidateDatasetJsonCommand(DataverseRequest aRequest, Dataverse target, S
 
     @Override
     public String execute(CommandContext ctxt) throws CommandException {
-        try {
-            String retVal = ctxt.dataverses().isDatasetJsonValid(dataverse.getAlias(), datasetJson);
-            return retVal;
-        } catch (Exception ex) {
-            String error = "Exception caught in getting the schema for this collection. Error: " + ex;
-            return error;
-        }
+
+            return ctxt.dataverses().isDatasetJsonValid(dataverse.getAlias(), datasetJson);
+
     }   
 }

From 2ce299738bb4aa121e09af6ea18bb30662f2f72a Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Thu, 9 Nov 2023 17:13:46 -0500
Subject: [PATCH 698/922] Fix ProvIT enabling the provenance feature

---
 src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
index 9b3b66538d7..3bfa3d72fbd 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
@@ -17,6 +17,8 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+
 public class ProvIT {
     
     @BeforeAll
@@ -27,6 +29,8 @@ public static void setUpClass() {
     
     @Test
     public void testFreeformDraftActions() {
+
+        UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();
         createDepositor.then().assertThat()
@@ -71,6 +75,7 @@ public void testFreeformDraftActions() {
         JsonObject provFreeFormGood = Json.createObjectBuilder()
                 .add("text", "I inherited this file from my grandfather.")
                 .build();
+        
         Response uploadProvFreeForm = UtilIT.uploadProvFreeForm(dataFileId.toString(), provFreeFormGood, apiTokenForDepositor);
         uploadProvFreeForm.prettyPrint();
         uploadProvFreeForm.then().assertThat()
@@ -81,12 +86,15 @@ public void testFreeformDraftActions() {
         datasetVersions.then().assertThat()
                 .body("data[0].versionState", equalTo("DRAFT"));
         
+        UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
         
     }
     
     @Test
     public void testAddProvFile() {
 
+        UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();
         createDepositor.then().assertThat()
@@ -196,6 +204,7 @@ public void testAddProvFile() {
                 .body("data.json", notNullValue(String.class));
         assertEquals(200, getProvJson.getStatusCode());
         
+        
         // TODO: Test that if provenance already exists in CPL (e.g. cplId in fileMetadata is not 0) upload returns error.
         //       There are currently no api endpoints to set up up this test.
         
@@ -204,6 +213,7 @@ public void testAddProvFile() {
         deleteProvJson.then().assertThat()
                 .statusCode(FORBIDDEN.getStatusCode()); //cannot delete json of a published dataset
 
+        UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
 // Command removed, redundant        
 //        Response deleteProvFreeForm = UtilIT.deleteProvFreeForm(dataFileId.toString(), apiTokenForDepositor);
 //        deleteProvFreeForm.prettyPrint();

From 6485444c26ecc2bab226aae75c7ee99f0b1b45e6 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Fri, 10 Nov 2023 15:26:02 +0100
Subject: [PATCH 699/922] single retry when datafile checksum validation fails

---
 .../harvard/iq/dataverse/util/FileUtil.java   | 30 ++++++++++++++-----
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 1ad389fb0e2..8e408e51f39 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1449,6 +1449,16 @@ public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	return s3io;
     }
     
+    private static InputStream getInputStream(StorageIO<DataFile> storage, boolean isTabularData) throws IOException {
+        if (!isTabularData) {
+            return storage.getInputStream();
+        } else {
+            // if this is a tabular file, read the preserved original "auxiliary file"
+            // instead:
+            return storage.getAuxFileAsInputStream(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);
+        }
+    }
+
     public static void validateDataFileChecksum(DataFile dataFile) throws IOException {
         DataFile.ChecksumType checksumType = dataFile.getChecksumType();
         if (checksumType == null) {
@@ -1462,14 +1472,7 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
 
         try {
             storage.open(DataAccessOption.READ_ACCESS);
-
-            if (!dataFile.isTabularData()) {
-                in = storage.getInputStream();
-            } else {
-                // if this is a tabular file, read the preserved original "auxiliary file"
-                // instead:
-                in = storage.getAuxFileAsInputStream(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);
-            }
+            in = getInputStream(storage, dataFile.isTabularData());
         } catch (IOException ioex) {
             in = null;
         }
@@ -1484,7 +1487,18 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
         try {
             recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
         } catch (RuntimeException rte) {
+            logger.log(Level.SEVERE, "failed to calculated checksum, one retry", rte);
             recalculatedChecksum = null;
+            IOUtils.closeQuietly(in);
+            storage = dataFile.getStorageIO();
+            try {
+                storage.open(DataAccessOption.READ_ACCESS);
+                in = getInputStream(storage, dataFile.isTabularData());
+                recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
+            } catch (RuntimeException rte2) {
+                logger.log(Level.SEVERE, "failed to calculated checksum, no retry", rte2);
+                recalculatedChecksum = null;
+            }
         } finally {
             IOUtils.closeQuietly(in);
         }

From 160f9f77f7a958f28faa2bafdfff775b69dff695 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Fri, 10 Nov 2023 16:19:40 +0100
Subject: [PATCH 700/922] rename getInputStream -> getOriginalFileInputStream
 and moved storage.open inside that method

---
 .../java/edu/harvard/iq/dataverse/util/FileUtil.java     | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 8e408e51f39..4fdeed421f1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1449,7 +1449,8 @@ public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) {
     	return s3io;
     }
     
-    private static InputStream getInputStream(StorageIO<DataFile> storage, boolean isTabularData) throws IOException {
+    private static InputStream getOriginalFileInputStream(StorageIO<DataFile> storage, boolean isTabularData) throws IOException {
+        storage.open(DataAccessOption.READ_ACCESS);
         if (!isTabularData) {
             return storage.getInputStream();
         } else {
@@ -1471,8 +1472,7 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
         InputStream in = null;
 
         try {
-            storage.open(DataAccessOption.READ_ACCESS);
-            in = getInputStream(storage, dataFile.isTabularData());
+            in = getOriginalFileInputStream(storage, dataFile.isTabularData());
         } catch (IOException ioex) {
             in = null;
         }
@@ -1492,8 +1492,7 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
             IOUtils.closeQuietly(in);
             storage = dataFile.getStorageIO();
             try {
-                storage.open(DataAccessOption.READ_ACCESS);
-                in = getInputStream(storage, dataFile.isTabularData());
+                in = getOriginalFileInputStream(storage, dataFile.isTabularData());
                 recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
             } catch (RuntimeException rte2) {
                 logger.log(Level.SEVERE, "failed to calculated checksum, no retry", rte2);

From eee3d87716dab80a12e650a88497ecfb1d8ecb07 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Fri, 10 Nov 2023 16:41:31 +0100
Subject: [PATCH 701/922] try-with-resources

---
 .../harvard/iq/dataverse/util/FileUtil.java   | 29 +++++--------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 4fdeed421f1..26cfb97a0b6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1469,37 +1469,24 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
         }
 
         StorageIO<DataFile> storage = dataFile.getStorageIO();
-        InputStream in = null;
+        String recalculatedChecksum = null;
 
-        try {
-            in = getOriginalFileInputStream(storage, dataFile.isTabularData());
+        try (InputStream inputStream = getOriginalFileInputStream(storage, dataFile.isTabularData())) {
+            recalculatedChecksum = FileUtil.calculateChecksum(inputStream, checksumType);
         } catch (IOException ioex) {
-            in = null;
-        }
-
-        if (in == null) {
             String info = BundleUtil.getStringFromBundle("dataset.publish.file.validation.error.failRead", Arrays.asList(dataFile.getId().toString()));
             logger.log(Level.INFO, info);
             throw new IOException(info);
-        }
-
-        String recalculatedChecksum = null;
-        try {
-            recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
         } catch (RuntimeException rte) {
             logger.log(Level.SEVERE, "failed to calculated checksum, one retry", rte);
             recalculatedChecksum = null;
-            IOUtils.closeQuietly(in);
+        }
+
+        if (recalculatedChecksum == null) { //retry once
             storage = dataFile.getStorageIO();
-            try {
-                in = getOriginalFileInputStream(storage, dataFile.isTabularData());
-                recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
-            } catch (RuntimeException rte2) {
-                logger.log(Level.SEVERE, "failed to calculated checksum, no retry", rte2);
-                recalculatedChecksum = null;
+            try (InputStream inputStream = getOriginalFileInputStream(storage, dataFile.isTabularData())) {
+                recalculatedChecksum = FileUtil.calculateChecksum(inputStream, checksumType);
             }
-        } finally {
-            IOUtils.closeQuietly(in);
         }
 
         if (recalculatedChecksum == null) {

From 50c3620106b1b4ab98242c9d9e4e2d9914d70d48 Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Fri, 10 Nov 2023 16:45:05 +0100
Subject: [PATCH 702/922] fixed compile error

---
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 26cfb97a0b6..6c540e88fb3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1504,6 +1504,7 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
             boolean fixed = false;
             if (!dataFile.isTabularData() && dataFile.getIngestReport() != null) {
                 // try again, see if the .orig file happens to be there:
+                InputStream in = null;
                 try {
                     in = storage.getAuxFileAsInputStream(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);
                 } catch (IOException ioex) {

From 7715ff9aa78364d2d49296ceb079145838cb401c Mon Sep 17 00:00:00 2001
From: Eryk Kulikowski <eryk.kulikowski@kuleuven.be>
Date: Fri, 10 Nov 2023 16:49:09 +0100
Subject: [PATCH 703/922] try-with-resources

---
 .../harvard/iq/dataverse/util/FileUtil.java    | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index 6c540e88fb3..df0c3e5a019 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1504,20 +1504,12 @@ public static void validateDataFileChecksum(DataFile dataFile) throws IOExceptio
             boolean fixed = false;
             if (!dataFile.isTabularData() && dataFile.getIngestReport() != null) {
                 // try again, see if the .orig file happens to be there:
-                InputStream in = null;
-                try {
-                    in = storage.getAuxFileAsInputStream(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION);
-                } catch (IOException ioex) {
-                    in = null;
+                try (InputStream in = storage.getAuxFileAsInputStream(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION)) {
+                    recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
+                } catch (RuntimeException rte) {
+                    recalculatedChecksum = null;
                 }
-                if (in != null) {
-                    try {
-                        recalculatedChecksum = FileUtil.calculateChecksum(in, checksumType);
-                    } catch (RuntimeException rte) {
-                        recalculatedChecksum = null;
-                    } finally {
-                        IOUtils.closeQuietly(in);
-                    }
+                if (recalculatedChecksum != null) {
                     // try again:
                     if (recalculatedChecksum.equals(dataFile.getChecksumValue())) {
                         fixed = true;

From aa7eceeb762eca045127cf91acb35d6c62b00d79 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 09:43:52 -0500
Subject: [PATCH 704/922] add return null if commandexception

---
 src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
index 9fb584a9133..8b09291d052 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
@@ -320,7 +320,7 @@ public String save() {
             logger.info("Guestbook Page Command Exception. Dataverse: " + dataverse.getName());
             logger.info(ex.toString());
             FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("guestbook.save.fail"), " - " + ex.toString()));
-            //logger.severe(ex.getMessage());
+            return null;
         }
         editMode = null;
         String msg = (create)? BundleUtil.getStringFromBundle("guestbook.create"): BundleUtil.getStringFromBundle("guestbook.save");

From ba4d178f5c541ec88ea0879ec5c715bda529f2c9 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 11:25:30 -0500
Subject: [PATCH 705/922] allow longer custom questions

---
 src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java
index 2cb6f27c3e4..d880da5b4a8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java
@@ -2,7 +2,7 @@
 import java.io.Serializable;
 import java.util.List;
 import jakarta.persistence.*;
-import org.hibernate.validator.constraints.NotBlank;
+import jakarta.validation.constraints.NotBlank;
 
 /**
  *
@@ -41,7 +41,7 @@ public void setId(Long id) {
     private String questionType;
     
     @NotBlank(message = "{custom.questiontext}")
-    @Column( nullable = false )
+    @Column( nullable = false, columnDefinition = "TEXT")
     private String questionString;
     private boolean required;
     

From 6cb1b9a961e5c4c5179f425c4c89dc478c29341a Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 11:25:48 -0500
Subject: [PATCH 706/922] fix gb error message display

---
 src/main/webapp/resources/iqbs/messages.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/resources/iqbs/messages.xhtml b/src/main/webapp/resources/iqbs/messages.xhtml
index bd17cf34d21..f8e1f5e8e9d 100644
--- a/src/main/webapp/resources/iqbs/messages.xhtml
+++ b/src/main/webapp/resources/iqbs/messages.xhtml
@@ -63,7 +63,7 @@
                         <f:convertDateTime pattern="yyyy-MM-dd HH:mm:ss" type="date" />
                     </h:outputText>
                     <strong>Server: </strong>
-                    #{systemConfig.dataverseServer}
+                    #{systemConfig.dataverseSiteUrl}
                 </ui:fragment>
             </div>
             #{msg.rendered()}

From 1652764b44abbdd887a2957ec42ad31e108dd864 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 13:36:27 -0500
Subject: [PATCH 707/922] flyway script and note

---
 .../10117-support for longer custom questions in guestbooks.md   | 1 +
 .../V6.0.0.4__10017-failure-with-long-custom-question.sql        | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 doc/release-notes/10117-support for longer custom questions in guestbooks.md
 create mode 100644 src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql

diff --git a/doc/release-notes/10117-support for longer custom questions in guestbooks.md b/doc/release-notes/10117-support for longer custom questions in guestbooks.md
new file mode 100644
index 00000000000..ab5e84d78fe
--- /dev/null
+++ b/doc/release-notes/10117-support for longer custom questions in guestbooks.md	
@@ -0,0 +1 @@
+Custom questions in Guestbooks can now be more than 255 characters and the bug causing a silent failure when questions were longer than this limit has been fixed.
\ No newline at end of file
diff --git a/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql b/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql
new file mode 100644
index 00000000000..9a3002378b3
--- /dev/null
+++ b/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql
@@ -0,0 +1 @@
+ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text;

From 2424a55235cc015cfba283146dc2f97d2b8699af Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 13:38:23 -0500
Subject: [PATCH 708/922] Revert "flyway script and note"

This reverts commit 1652764b44abbdd887a2957ec42ad31e108dd864.
---
 .../10117-support for longer custom questions in guestbooks.md   | 1 -
 .../V6.0.0.4__10017-failure-with-long-custom-question.sql        | 1 -
 2 files changed, 2 deletions(-)
 delete mode 100644 doc/release-notes/10117-support for longer custom questions in guestbooks.md
 delete mode 100644 src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql

diff --git a/doc/release-notes/10117-support for longer custom questions in guestbooks.md b/doc/release-notes/10117-support for longer custom questions in guestbooks.md
deleted file mode 100644
index ab5e84d78fe..00000000000
--- a/doc/release-notes/10117-support for longer custom questions in guestbooks.md	
+++ /dev/null
@@ -1 +0,0 @@
-Custom questions in Guestbooks can now be more than 255 characters and the bug causing a silent failure when questions were longer than this limit has been fixed.
\ No newline at end of file
diff --git a/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql b/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql
deleted file mode 100644
index 9a3002378b3..00000000000
--- a/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql
+++ /dev/null
@@ -1 +0,0 @@
-ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text;

From 6fc0f358ae53fec619b93a6deafd9297ce6a9294 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 13:38:30 -0500
Subject: [PATCH 709/922] Revert "fix gb error message display"

This reverts commit 6cb1b9a961e5c4c5179f425c4c89dc478c29341a.
---
 src/main/webapp/resources/iqbs/messages.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/resources/iqbs/messages.xhtml b/src/main/webapp/resources/iqbs/messages.xhtml
index f8e1f5e8e9d..bd17cf34d21 100644
--- a/src/main/webapp/resources/iqbs/messages.xhtml
+++ b/src/main/webapp/resources/iqbs/messages.xhtml
@@ -63,7 +63,7 @@
                         <f:convertDateTime pattern="yyyy-MM-dd HH:mm:ss" type="date" />
                     </h:outputText>
                     <strong>Server: </strong>
-                    #{systemConfig.dataverseSiteUrl}
+                    #{systemConfig.dataverseServer}
                 </ui:fragment>
             </div>
             #{msg.rendered()}

From 00a17071c358b7ebee09e77130cb7319c665dfb5 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 13:38:36 -0500
Subject: [PATCH 710/922] Revert "allow longer custom questions"

This reverts commit ba4d178f5c541ec88ea0879ec5c715bda529f2c9.
---
 src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java
index d880da5b4a8..2cb6f27c3e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java
+++ b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java
@@ -2,7 +2,7 @@
 import java.io.Serializable;
 import java.util.List;
 import jakarta.persistence.*;
-import jakarta.validation.constraints.NotBlank;
+import org.hibernate.validator.constraints.NotBlank;
 
 /**
  *
@@ -41,7 +41,7 @@ public void setId(Long id) {
     private String questionType;
     
     @NotBlank(message = "{custom.questiontext}")
-    @Column( nullable = false, columnDefinition = "TEXT")
+    @Column( nullable = false )
     private String questionString;
     private boolean required;
     

From d3fbee58262ac439a0b10f4ca7e1494dea4a6c5d Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 13:38:43 -0500
Subject: [PATCH 711/922] Revert "add return null if commandexception"

This reverts commit aa7eceeb762eca045127cf91acb35d6c62b00d79.
---
 src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
index 8b09291d052..9fb584a9133 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java
@@ -320,7 +320,7 @@ public String save() {
             logger.info("Guestbook Page Command Exception. Dataverse: " + dataverse.getName());
             logger.info(ex.toString());
             FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("guestbook.save.fail"), " - " + ex.toString()));
-            return null;
+            //logger.severe(ex.getMessage());
         }
         editMode = null;
         String msg = (create)? BundleUtil.getStringFromBundle("guestbook.create"): BundleUtil.getStringFromBundle("guestbook.save");

From 4b347c7ec13591ba38ffa55fbde394cce2b8bcfe Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 17:47:17 -0500
Subject: [PATCH 712/922] doc update

---
 .../source/developers/big-data-support.rst    | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
index 1917967b3f3..d38f7f27a68 100644
--- a/doc/sphinx-guides/source/developers/big-data-support.rst
+++ b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -149,20 +149,30 @@ Globus File Transfer
 
 Note: Globus file transfer is still experimental but feedback is welcome! See :ref:`support`.
 
-Users can transfer files via `Globus <ttps://www.globus.org>`_ into and out of datasets when their Dataverse installation is configured to use a Globus accessible S3 store and a community-developed `dataverse-globus <https://github.com/scholarsportal/dataverse-globus>`_ "transfer" app has been properly installed and configured.
+Users can transfer files via `Globus <ttps://www.globus.org>`_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s) 
+and a community-developed `dataverse-globus <https://github.com/scholarsportal/dataverse-globus>`_ app has been properly installed and configured.
 
 Due to differences in the access control models of a Dataverse installation and Globus, enabling the Globus capability on a store will disable the ability to restrict and embargo files in that store.
 
-As Globus aficionados know, Globus endpoints can be in a variety of places, from data centers to personal computers. This means that from within the Dataverse software, a Globus transfer can feel like an upload or a download (with Globus Personal Connect running on your laptop, for example) or it can feel like a true transfer from one server to another (from a cluster in a data center into a Dataverse dataset or vice versa).
+Globus endpoints can be in a variety of places, from data centers to personal computers. 
+This means that from within the Dataverse software, a Globus transfer can feel like an upload or a download (with Globus Personal Connect running on your laptop, for example) or it can feel like a true transfer from one server to another (from a cluster in a data center into a Dataverse dataset or vice versa).
 
-Globus transfer uses a very efficient transfer mechanism and has additional features that make it suitable for large files and large numbers of files:
+Globus transfer uses an efficient transfer mechanism and has additional features that make it suitable for large files and large numbers of files:
 
 * robust file transfer capable of restarting after network or endpoint failures
 * third-party transfer, which enables a user accessing a Dataverse installation in their desktop browser to initiate transfer of their files from a remote endpoint (i.e. on a local high-performance computing cluster), directly to an S3 store managed by the Dataverse installation
 
-Globus transfer requires use of the Globus S3 connector which requires a paid Globus subscription at the host institution. Users will need a Globus account which could be obtained via their institution or directly from Globus (at no cost).
+Dataverse supports three options for using Globus, two involving transfer to Dataverse-managed endpoints and one allowing Dataverse to reference files on remote endpoints.
+Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus
+S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint.
+Users will need a Globus account, which can be obtained via their institution or directly from Globus (at no cost).
 
-The setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document <https://docs.google.com/document/d/1mwY3IVv8_wTspQC0d4ddFrD2deqwr-V5iAGHgOy4Ch8/edit?usp=sharing>`_ and the references therein.
+For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. 
+Users will need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described in the dataset's Terms of Use and Access).
+
+All of Dataverse's Globus capabilities are now store-based (see the store documentation) and therefore different collections/datasets can be configured to use different Globus-capable stores (or normal file, S3 stores, etc.)
+
+More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document <https://docs.google.com/document/d/1mwY3IVv8_wTspQC0d4ddFrD2deqwr-V5iAGHgOy4Ch8/edit?usp=sharing>`_ and the references therein.
 
 As described in that document, Globus transfers can be initiated by choosing the Globus option in the dataset upload panel. (Globus, which does asynchronous transfers, is not available during dataset creation.) Analogously, "Globus Transfer" is one of the download options in the "Access Dataset" menu and optionally the file landing page download menu (if/when supported in the dataverse-globus app).
 

From 6ad55eb689071921857a9f97135e97dd2e71c076 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 17:50:16 -0500
Subject: [PATCH 713/922] Support multiple ref endpoints for non-managed case

---
 .../harvard/iq/dataverse/api/Datasets.java    |  72 ++++----
 .../dataaccess/GlobusAccessibleStore.java     |  14 +-
 .../dataaccess/GlobusOverlayAccessIO.java     | 166 +++++++++++++-----
 .../dataaccess/RemoteOverlayAccessIO.java     |  47 +++--
 .../dataverse/globus/GlobusServiceBean.java   |  31 +++-
 5 files changed, 226 insertions(+), 104 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index b1c528f3fd9..a57f373f106 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3601,13 +3601,11 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         }
 
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
-        if (managed) {
-
-            allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
+        String requestCallName = managed ? "requestGlobusTransferPaths" : "requestGlobusReferencePaths";
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, requestCallName)
                     .add(URLTokenUtil.HTTP_METHOD, "POST")
-                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusTransferPaths")
+                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusPaths")
                     .add(URLTokenUtil.TIMEOUT, 300));
-        }
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addGlobusFiles")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
@@ -3632,7 +3630,7 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
      */
     @POST
     @AuthRequired
-    @Path("{id}/requestGlobusTransferPaths")
+    @Path("{id}/requestGlobusPaths")
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
     public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, String jsonBody
@@ -3666,35 +3664,45 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
-
-        if(!GlobusAccessibleStore.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
-            return badRequest("This dataset does not have managed Globus storage");
-        }
-            
         if (permissionSvc.requestOn(createDataverseRequest(authUser), dataset)
                 .canIssue(UpdateDatasetVersionCommand.class)) {
-            try {
+
             JsonObject params = JsonUtil.getJsonObject(jsonBody);
-            String principal = params.getString("principal");
-            int numberOfPaths = params.getInt("numberOfFiles");
-            if(numberOfPaths <=0) {
-                return badRequest("numberOfFiles must be positive");
-            }
-
-            JsonObject response = globusService.requestAccessiblePaths(principal, dataset, numberOfPaths);
-            switch (response.getInt("status")) {
-            case 201:
-                return ok(response.getJsonObject("paths"));
-            case 400:
-                return badRequest("Unable to grant permission");
-            case 409:
-                return conflict("Permission already exists");
-            default:
-                return error(null, "Unexpected error when granting permission");
-            }
-            } catch (NullPointerException|ClassCastException e) {
-                return badRequest("Error retrieving principal and numberOfFiles from JSON request body");
-                
+            if (!GlobusAccessibleStore.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
+                try {
+                    JsonArray referencedFiles = params.getJsonArray("referencedFiles");
+                    if (referencedFiles == null || referencedFiles.size() == 0) {
+                        return badRequest("No referencedFiles specified");
+                    }
+                    JsonObject fileMap = globusService.requestReferenceFileIdentifiers(dataset, referencedFiles);
+                    return (ok(fileMap));
+                } catch (Exception e) {
+                    return badRequest(e.getLocalizedMessage());
+                }
+            } else {
+                try {
+                    String principal = params.getString("principal");
+                    int numberOfPaths = params.getInt("numberOfFiles");
+                    if (numberOfPaths <= 0) {
+                        return badRequest("numberOfFiles must be positive");
+                    }
+
+                    JsonObject response = globusService.requestAccessiblePaths(principal, dataset, numberOfPaths);
+                    switch (response.getInt("status")) {
+                    case 201:
+                        return ok(response.getJsonObject("paths"));
+                    case 400:
+                        return badRequest("Unable to grant permission");
+                    case 409:
+                        return conflict("Permission already exists");
+                    default:
+                        return error(null, "Unexpected error when granting permission");
+                    }
+
+                } catch (NullPointerException | ClassCastException e) {
+                    return badRequest("Error retrieving principal and numberOfFiles from JSON request body");
+
+                }
             }
         } else {
             return forbidden("User doesn't have permission to upload to this dataset");
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index 1d98044b2b5..afc7556481a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -7,8 +7,7 @@
 public interface GlobusAccessibleStore {
 
     static final String MANAGED = "managed";
-    static final String GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH = "globus-transfer-endpoint-with-basepath";
-    static final String GLOBUS_REFERENCE_ENDPOINTS_WITH_BASEPATHS = "globus-reference-endpoints-with-basepaths";
+    static final String TRANSFER_ENDPOINT_WITH_BASEPATH = "transfer-endpoint-with-basepath";
     static final String GLOBUS_TOKEN = "globus-token";
     
     public static boolean isDataverseManaged(String driverId) {
@@ -16,37 +15,36 @@ public static boolean isDataverseManaged(String driverId) {
     }
     
     public static String getTransferEndpointId(String driverId) {
-        String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
+        String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, TRANSFER_ENDPOINT_WITH_BASEPATH);
         int pathStart = endpointWithBasePath.indexOf("/");
         return pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
     }
     
     public static String getTransferPath(String driverId) {
-        String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH);
+        String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, TRANSFER_ENDPOINT_WITH_BASEPATH);
         int pathStart = endpointWithBasePath.indexOf("/");
         return pathStart > 0 ? endpointWithBasePath.substring(pathStart) : "";
 
     }
 
     public static JsonArray getReferenceEndpointsWithPaths(String driverId) {
-        String[] endpoints = StorageIO.getConfigParamForDriver(driverId, GLOBUS_REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*");
+        String[] endpoints = StorageIO.getConfigParamForDriver(driverId, RemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*");
         JsonArrayBuilder builder = Json.createArrayBuilder();
         for(int i=0;i<endpoints.length;i++) {
             builder.add(endpoints[i]);
         }
         return builder.build();
     }
-
     
     public static boolean acceptsGlobusTransfers(String storeId) {
-        if(StorageIO.getConfigParamForDriver(storeId, GLOBUS_TRANSFER_ENDPOINT_WITH_BASEPATH) != null) {
+        if(StorageIO.getConfigParamForDriver(storeId, TRANSFER_ENDPOINT_WITH_BASEPATH) != null) {
             return true;
         }
         return false;
     }
 
     public static boolean allowsGlobusReferences(String storeId) {
-        if(StorageIO.getConfigParamForDriver(storeId, GLOBUS_REFERENCE_ENDPOINTS_WITH_BASEPATHS) != null) {
+        if(StorageIO.getConfigParamForDriver(storeId, RemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS) != null) {
             return true;
         }
         return false;
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 97884bd5722..0dec7133fb5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -12,6 +12,8 @@
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.logging.Logger;
 
@@ -39,7 +41,8 @@
  * <globusEndpointId/basepath>/<dataset authority>/<dataset
  * identifier>/<baseStorageIdentifier>
  *
- * baseUrl: globus://<globusEndpointId/basePath>
+ * transfer and reference endpoint formats: <globusEndpointId/basePath>
+ * reference endpoints separated by a comma
  * 
  */
 public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> implements GlobusAccessibleStore {
@@ -50,7 +53,7 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
      * Dataverse/the globus app manage file locations, access controls, deletion,
      * etc.
      */
-    private boolean dataverseManaged = false;
+    private Boolean dataverseManaged = null;
 
     private String relativeDirectoryPath;
     
@@ -58,22 +61,59 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
     
     private String filename;
 
+    private String[] allowedEndpoints;
     private String endpoint;
 
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
-        dataverseManaged = GlobusAccessibleStore.isDataverseManaged(this.driverId);
     }
 
+
+    public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
+        this.driverId = driverId;
+        configureStores(null, driverId, storageLocation);
+        if (isManaged()) {
+            String[] parts = DataAccess.getDriverIdAndStorageLocation(storageLocation);
+            path = parts[1];
+        } else {
+            this.setIsLocalFile(false);
+            path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
+            validatePath(path);
+            logger.fine("Referenced path: " + path);
+        }
+    }
+    private boolean isManaged() {
+        if(dataverseManaged==null) {
+            dataverseManaged = GlobusAccessibleStore.isDataverseManaged(this.driverId);
+        }
+        return dataverseManaged;
+    }
+    
+    private String retrieveGlobusAccessToken() {
+        String globusToken = getConfigParam(GlobusAccessibleStore.GLOBUS_TOKEN);
+        
+
+        AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
+        return accessToken.getOtherTokens().get(0).getAccessToken();
+    }
+
+
     private void parsePath() {
         int filenameStart = path.lastIndexOf("/") + 1;
-        String endpointWithBasePath = baseUrl.substring(baseUrl.lastIndexOf(DataAccess.SEPARATOR) + 3);
+        String endpointWithBasePath = null;
+        if (!isManaged()) {
+            endpointWithBasePath = findMatchingEndpoint(path, allowedEndpoints);
+        } else {
+            endpointWithBasePath = allowedEndpoints[0];
+        }
+        //String endpointWithBasePath = baseEndpointPath.substring(baseEndpointPath.lastIndexOf(DataAccess.SEPARATOR) + 3);
         int pathStart = endpointWithBasePath.indexOf("/");
         logger.info("endpointWithBasePath: " + endpointWithBasePath);
         endpointPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
         logger.info("endpointPath: " + endpointPath);
+        
 
-        if (dataverseManaged && (dvObject!=null)) {
+        if (isManaged() && (dvObject!=null)) {
             
             Dataset ds = null;
             if (dvObject instanceof Dataset) {
@@ -95,40 +135,36 @@ private void parsePath() {
         
     }
 
-    public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
-        this.driverId = driverId;
-        configureStores(null, driverId, storageLocation);
-        this.dataverseManaged = GlobusAccessibleStore.isDataverseManaged(this.driverId);
-        if (dataverseManaged) {
-            String[] parts = DataAccess.getDriverIdAndStorageLocation(storageLocation);
-            path = parts[1];
-        } else {
-            this.setIsLocalFile(false);
-            path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
-            validatePath(path);
-            logger.fine("Relative path: " + path);
+    private static String findMatchingEndpoint(String path, String[] allowedEndpoints) {
+        for(int i=0;i<allowedEndpoints.length;i++) {
+            if (path.startsWith(allowedEndpoints[i])) {
+                return allowedEndpoints[i];
+            }
         }
+        logger.warning("Could not find matching endpoint for path: " + path);
+        return null;
     }
-    
-    private String retrieveGlobusAccessToken() {
-        String globusToken = getConfigParam(GlobusAccessibleStore.GLOBUS_TOKEN);
-        
-
-        AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
-        return accessToken.getOtherTokens().get(0).getAccessToken();
-    }
-
 
+    @Override
+    protected void validatePath(String relPath) throws IOException {
+        if (isManaged()) {
+            if (!usesStandardNamePattern(relPath)) {
+                throw new IOException("Unacceptable identifier pattern in submitted identifier: " + relPath);
+            }
+        } else {
+            try {
+                String endpoint = findMatchingEndpoint(relPath, allowedEndpoints);
+                logger.info(endpoint + "  " + relPath);
 
-    private void validatePath(String relPath) throws IOException {
-        try {
-            URI absoluteURI = new URI(baseUrl + "/" + relPath);
-            if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
-                throw new IOException("storageidentifier doesn't start with " + this.driverId + "'s base-url");
+                if (endpoint == null || !Paths.get(endpoint, relPath).normalize().startsWith(endpoint)) {
+                    throw new IOException(
+                            "storageidentifier doesn't start with one of " + this.driverId + "'s allowed endpoints");
+                }
+            } catch (InvalidPathException e) {
+                throw new IOException("Could not interpret storageidentifier in globus store " + this.driverId);
             }
-        } catch (URISyntaxException use) {
-            throw new IOException("Could not interpret storageidentifier in remote store " + this.driverId);
         }
+
     }
 
     // Call the Globus API to get the file size
@@ -245,7 +281,7 @@ public void delete() throws IOException {
             }
         } catch (Exception e) {
             logger.warning(e.getMessage());
-            throw new IOException("Error deleting: " + baseUrl + "/" + path);
+            throw new IOException("Error deleting: " + endpoint + "/" + path);
 
         }
 
@@ -258,13 +294,14 @@ public void delete() throws IOException {
     public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliaryType, String auxiliaryFileName)
             throws IOException {
 //Fix
+        parsePath();
         // ToDo - support remote auxiliary Files
         if (auxiliaryTag == null) {
             String secretKey = getConfigParam(SECRET_KEY);
             if (secretKey == null) {
-                return baseUrl + "/" + path;
+                return endpoint + "/" + path;
             } else {
-                return UrlSignerUtil.signUrl(baseUrl + "/" + path, getUrlExpirationMinutes(), null, "GET", secretKey);
+                return UrlSignerUtil.signUrl(endpoint + "/" + path, getUrlExpirationMinutes(), null, "GET", secretKey);
             }
         } else {
             return baseStore.generateTemporaryDownloadUrl(auxiliaryTag, auxiliaryType, auxiliaryFileName);
@@ -273,10 +310,10 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
 
     static boolean isValidIdentifier(String driverId, String storageId) {
         String baseIdentifier = storageId.substring(storageId.lastIndexOf("//") + 2);
-        String baseUrl = getConfigParamForDriver(driverId, BASE_URL);
-        if (baseUrl == null) {
-            return false;
-        }
+        try {
+            
+        String[] allowedEndpoints =getAllowedEndpoints(driverId);
+            
         // Internally managed endpoints require standard name pattern (submitted via
         // /addFile(s) api)
         if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
@@ -290,30 +327,65 @@ static boolean isValidIdentifier(String driverId, String storageId) {
         }
         // Remote endpoints require a valid URI within the baseUrl
         try {
-            URI absoluteURI = new URI(baseUrl + "/" + baseIdentifier);
-            if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
-                logger.warning("storageidentifier doesn't start with " + driverId + "'s base-url: " + storageId);
+            String endpoint = findMatchingEndpoint(baseIdentifier, allowedEndpoints);
+            
+            if(endpoint==null || !Paths.get(endpoint, baseIdentifier).normalize().startsWith(endpoint)) {
+                logger.warning("storageidentifier doesn't start with one of " + driverId + "'s allowed endpoints");
                 return false;
             }
-        } catch (URISyntaxException use) {
-            logger.warning("Could not interpret storageidentifier in remote store " + driverId + " : " + storageId);
-            logger.warning(use.getLocalizedMessage());
+        } catch (InvalidPathException e) {
+            logger.warning("Could not interpret storageidentifier in globus store " + driverId);
             return false;
         }
         return true;
+        } catch (IOException e) {
+            return false;
+        }
+        
     }
 
     @Override
     public String getStorageLocation() throws IOException {
         parsePath();
-        if (dataverseManaged) {
+        if (isManaged()) {
             return this.driverId + DataAccess.SEPARATOR + relativeDirectoryPath + "/" + filename;
         } else {
             return super.getStorageLocation();
         }
     }
     
+    /** This endpoint configures all the endpoints the store is allowed to reference data from. At present, the RemoteOverlayAccessIO only supports a single endpoint but
+     * the derived GlobusOverlayAccessIO can support multiple endpoints.
+     * @throws IOException
+     */
+    @Override
+    protected void configureEndpoints() throws IOException {
+        allowedEndpoints = getAllowedEndpoints(this.driverId);
+        logger.info("Set allowed endpoints: " + Arrays.toString(allowedEndpoints));
+    }
     
+    private static String[] getAllowedEndpoints(String driverId) throws IOException {
+        String[] allowedEndpoints = null;
+        if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
+            allowedEndpoints = new String[1];
+            allowedEndpoints[0] = getConfigParamForDriver(driverId, TRANSFER_ENDPOINT_WITH_BASEPATH);
+            if (allowedEndpoints[0] == null) {
+                throw new IOException(
+                        "dataverse.files." + driverId + "." + TRANSFER_ENDPOINT_WITH_BASEPATH + " is required");
+            }
+        } else {
+            String rawEndpoints = getConfigParamForDriver(driverId, REFERENCE_ENDPOINTS_WITH_BASEPATHS);
+            if (rawEndpoints != null) {
+                allowedEndpoints = getConfigParamForDriver(driverId, REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*");
+            }
+            if (rawEndpoints == null || allowedEndpoints.length == 0) {
+                throw new IOException("dataverse.files." + driverId + ".base-url is required");
+            }
+        }
+        return allowedEndpoints;
+    }
+
+
     public static void main(String[] args) {
         System.out.println("Running the main method");
         if (args.length > 0) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index aafab038ae2..5463254140d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -65,7 +65,10 @@ public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
 
+    // A single baseUrl of the form http(s)://<host(:port)/basePath> where this store can reference data
     static final String BASE_URL = "base-url";
+    // Multiple endpoints where data can be referenced from. Multiple endpoints are separated by a comma. Multiple endpoints are only supported by the GlobalOverlayAccessIO at present. 
+    static final String REFERENCE_ENDPOINTS_WITH_BASEPATHS = "reference-endpoints-with-basepaths";
     static final String BASE_STORE = "base-store";
     static final String SECRET_KEY = "secret-key";
     static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
@@ -74,7 +77,7 @@ public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
     
     protected StorageIO<DvObject> baseStore = null;
     protected String path = null;
-    protected String baseUrl = null;
+    private String baseUrl = null;
 
     protected static HttpClientContext localContext = HttpClientContext.create();
     protected PoolingHttpClientConnectionManager cm = null;
@@ -110,7 +113,7 @@ public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOE
         logger.fine("Relative path: " + path);
     }
 
-    private void validatePath(String relPath) throws IOException {
+    protected void validatePath(String relPath) throws IOException {
         try {
             URI absoluteURI = new URI(baseUrl + "/" + relPath);
             if (!absoluteURI.normalize().toString().startsWith(baseUrl)) {
@@ -457,19 +460,8 @@ int getUrlExpirationMinutes() {
     }
 
     protected void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
-        baseUrl = getConfigParam(BASE_URL);
-        if (baseUrl == null) {
-            throw new IOException("dataverse.files." + this.driverId + ".base-url is required");
-        } else {
-            try {
-                new URI(baseUrl);
-            } catch (Exception e) {
-                logger.warning(
-                        "Trouble interpreting base-url for store: " + this.driverId + " : " + e.getLocalizedMessage());
-                throw new IOException("Can't interpret base-url as a URI");
-            }
-
-        }
+        configureEndpoints();
+        
 
         if (baseStore == null) {
             String baseDriverId = getBaseStoreIdFor(driverId);
@@ -543,6 +535,31 @@ protected void configureStores(DataAccessRequest req, String driverId, String st
         }
     }
 
+    /** This endpoint configures all the endpoints the store is allowed to reference data from. At present, the RemoteOverlayAccessIO only supports a single endpoint but
+     * the derived GlobusOverlayAccessIO can support multiple endpoints.
+     * @throws IOException
+     */
+    protected void configureEndpoints() throws IOException {
+        baseUrl = getConfigParam(BASE_URL);
+        if (baseUrl == null) {
+            //Will accept the first endpoint using the newer setting
+            baseUrl = getConfigParam(REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*")[0];
+            if (baseUrl == null) {
+                throw new IOException("dataverse.files." + this.driverId + ".base-url is required");
+            }
+        }
+        if (baseUrl != null) {
+            try {
+                new URI(baseUrl);
+            } catch (Exception e) {
+                logger.warning(
+                        "Trouble interpreting base-url for store: " + this.driverId + " : " + e.getLocalizedMessage());
+                throw new IOException("Can't interpret base-url as a URI");
+            }
+
+        }
+    }
+
     // Convenience method to assemble the path, starting with the DOI
     // authority/identifier/, that is needed to create a base store via
     // DataAccess.getDirectStorageIO - the caller has to add the store type specific
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index dab0e36852c..3dee3bd498f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -6,7 +6,6 @@
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.GsonBuilder;
 import edu.harvard.iq.dataverse.*;
-
 import jakarta.ejb.Asynchronous;
 import jakarta.ejb.EJB;
 import jakarta.ejb.Stateless;
@@ -20,6 +19,8 @@
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
 import jakarta.json.JsonPatch;
+import jakarta.json.JsonString;
+import jakarta.json.JsonValue.ValueType;
 import jakarta.json.stream.JsonParsingException;
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.ws.rs.HttpMethod;
@@ -57,7 +58,6 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
-import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -284,6 +284,33 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
             return response.build();
     }
 
+    public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray referencedFiles) {
+        String driverId = dataset.getEffectiveStorageDriverId();
+        JsonArray endpoints = GlobusAccessibleStore.getReferenceEndpointsWithPaths(driverId);
+
+        JsonObjectBuilder fileMap = Json.createObjectBuilder();
+        referencedFiles.forEach(value -> {
+            if (value.getValueType() != ValueType.STRING) {
+                throw new JsonParsingException("ReferencedFiles must be strings", null);
+            }
+            String referencedFile = ((JsonString) value).getString();
+            boolean valid = false;
+            for (int i = 0; i < endpoints.size(); i++) {
+                if (referencedFile.startsWith(((JsonString) endpoints.get(i)).getString())) {
+                    valid = true;
+                }
+            }
+            if (!valid) {
+                throw new IllegalArgumentException(
+                        "Referenced file " + referencedFile + " is not in an allowed endpoint/path");
+            }
+            String storageIdentifier = DataAccess.getNewStorageIdentifier(driverId);
+            fileMap.add(referencedFile,
+                    storageIdentifier + "//" + referencedFile);
+        });
+        return fileMap.build();
+    }
+
     //Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
             .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))

From 48f02dde7f22b21e28c8d635df904b79532f042a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 10 Nov 2023 17:56:56 -0500
Subject: [PATCH 714/922] handle file not found case

---
 .../iq/dataverse/dataaccess/GlobusOverlayAccessIO.java     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 0dec7133fb5..f42f5443108 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -25,6 +25,7 @@
 import org.apache.http.util.EntityUtils;
 
 import jakarta.json.Json;
+import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
 import jakarta.json.JsonObjectBuilder;
 
@@ -189,7 +190,11 @@ public long retrieveSizeFromMedia() {
                 String responseString = EntityUtils.toString(response.getEntity());
                 logger.info("Response from " + get.getURI().toString() + " is: " + responseString);
                 JsonObject responseJson = JsonUtil.getJsonObject(responseString);
-                return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getInt("size");
+                JsonArray dataArray = responseJson.getJsonArray("DATA");
+                if (dataArray != null && dataArray.size() != 0) {
+                    //File found
+                    return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getInt("size");
+                }
             } else {
                 logger.warning("Response from " + get.getURI().toString() + " was "
                         + response.getStatusLine().getStatusCode());

From c33f07aad938f4707e6985ddeeec801969e4a3fc Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Sat, 11 Nov 2023 14:38:00 -0500
Subject: [PATCH 715/922] Add logic to leave settings as found before test

---
 .../edu/harvard/iq/dataverse/api/ProvIT.java  | 30 +++++++++++--------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
index 3bfa3d72fbd..6b9b59f431d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
@@ -30,7 +30,12 @@ public static void setUpClass() {
     @Test
     public void testFreeformDraftActions() {
 
-        UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        Response provCollectionStatus = UtilIT.getSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        boolean provEnabled = provCollectionStatus.getStatusCode() == 200;
+        if(!provEnabled){
+            UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        }
+  
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();
         createDepositor.then().assertThat()
@@ -85,15 +90,20 @@ public void testFreeformDraftActions() {
         datasetVersions.prettyPrint();
         datasetVersions.then().assertThat()
                 .body("data[0].versionState", equalTo("DRAFT"));
-        
-        UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        
+        if(!provEnabled){
+            UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        }
+               
     }
     
     @Test
     public void testAddProvFile() {
 
-        UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        Response provCollectionStatus = UtilIT.getSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        boolean provEnabled = provCollectionStatus.getStatusCode() == 200;
+        if(!provEnabled){
+            UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        }
 
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();
@@ -213,12 +223,8 @@ public void testAddProvFile() {
         deleteProvJson.then().assertThat()
                 .statusCode(FORBIDDEN.getStatusCode()); //cannot delete json of a published dataset
 
-        UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-// Command removed, redundant        
-//        Response deleteProvFreeForm = UtilIT.deleteProvFreeForm(dataFileId.toString(), apiTokenForDepositor);
-//        deleteProvFreeForm.prettyPrint();
-//        deleteProvFreeForm.then().assertThat()
-//                .statusCode(OK.getStatusCode());
-        
+        if(!provEnabled){
+            UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        }
     }
 }

From 3407fb9f813984c857ef7708af7d6dc239b8f8ee Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 13 Nov 2023 07:04:15 -0500
Subject: [PATCH 716/922] Add ProvIT to integration-tests.txt

---
 tests/integration-tests.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt
index 18911b3164a..bb3bc7f9ce6 100644
--- a/tests/integration-tests.txt
+++ b/tests/integration-tests.txt
@@ -1 +1 @@
-DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT
+DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,ProvIT

From 2842cdaf246c531b04449ac4c8b20fc4a09c2668 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Mon, 13 Nov 2023 08:42:31 -0500
Subject: [PATCH 717/922] Move this change into BeforeAll/AfterAll

---
 .../edu/harvard/iq/dataverse/api/ProvIT.java  | 37 ++++++++++---------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
index 6b9b59f431d..69a87869fe1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
@@ -11,6 +11,9 @@
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.AfterAll;
+
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
 
@@ -20,22 +23,24 @@
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
 public class ProvIT {
+
+    private static boolean provEnabled = false;
     
     @BeforeAll
-    public static void setUpClass() {
+    public static void setUpClass() {        
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
+        Response provCollectionStatus = UtilIT.getSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        
+        provEnabled = provCollectionStatus.getStatusCode() == 200;
+        if(!provEnabled){
+            UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
+        }
     }
 
     
     @Test
     public void testFreeformDraftActions() {
 
-        Response provCollectionStatus = UtilIT.getSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        boolean provEnabled = provCollectionStatus.getStatusCode() == 200;
-        if(!provEnabled){
-            UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        }
-  
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();
         createDepositor.then().assertThat()
@@ -90,20 +95,11 @@ public void testFreeformDraftActions() {
         datasetVersions.prettyPrint();
         datasetVersions.then().assertThat()
                 .body("data[0].versionState", equalTo("DRAFT"));
-        if(!provEnabled){
-            UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        }
-               
+     
     }
     
     @Test
-    public void testAddProvFile() {
-
-        Response provCollectionStatus = UtilIT.getSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        boolean provEnabled = provCollectionStatus.getStatusCode() == 200;
-        if(!provEnabled){
-            UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        }
+    public void testAddProvFile() {        
 
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();
@@ -223,6 +219,11 @@ public void testAddProvFile() {
         deleteProvJson.then().assertThat()
                 .statusCode(FORBIDDEN.getStatusCode()); //cannot delete json of a published dataset
 
+        
+    }
+
+    @AfterAll
+    public static void tearDownClass() {
         if(!provEnabled){
             UtilIT.deleteSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
         }

From 437e7ccd480dbae405238faffb9fff8a8317218d Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 13 Nov 2023 09:56:16 -0500
Subject: [PATCH 718/922] #9464 remove unused import

---
 src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index fabb33e328a..557b7df202b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -128,7 +128,6 @@
 import java.util.Optional;
 import java.util.stream.Collectors;
 import jakarta.servlet.http.HttpServletResponse;
-import jakarta.validation.constraints.NotNull;
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.StreamingOutput;

From d029cacc9aae5e361869b73f7e76661c5ab8d549 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 13 Nov 2023 11:35:28 -0500
Subject: [PATCH 719/922] remove extra whitespace #10112

---
 src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
index 69a87869fe1..a944c6aa926 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
@@ -27,12 +27,12 @@ public class ProvIT {
     private static boolean provEnabled = false;
     
     @BeforeAll
-    public static void setUpClass() {        
+    public static void setUpClass() {
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
         Response provCollectionStatus = UtilIT.getSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
-        
+
         provEnabled = provCollectionStatus.getStatusCode() == 200;
-        if(!provEnabled){
+        if (!provEnabled) {
             UtilIT.enableSetting(SettingsServiceBean.Key.ProvCollectionEnabled);
         }
     }
@@ -99,7 +99,7 @@ public void testFreeformDraftActions() {
     }
     
     @Test
-    public void testAddProvFile() {        
+    public void testAddProvFile() {
 
         Response createDepositor = UtilIT.createRandomUser();
         createDepositor.prettyPrint();

From c09034d638147c5cd618e5ff4a460e1840b8cd0a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 13 Nov 2023 11:37:16 -0500
Subject: [PATCH 720/922] organize imports #10112

---
 .../java/edu/harvard/iq/dataverse/api/ProvIT.java  | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
index a944c6aa926..33323ff4239 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/ProvIT.java
@@ -1,27 +1,23 @@
 package edu.harvard.iq.dataverse.api;
 
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import io.restassured.RestAssured;
 import io.restassured.path.json.JsonPath;
 import io.restassured.response.Response;
 import jakarta.json.Json;
 import jakarta.json.JsonArray;
 import jakarta.json.JsonObject;
-import static jakarta.ws.rs.core.Response.Status.CREATED;
-import static jakarta.ws.rs.core.Response.Status.OK;
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
+import static jakarta.ws.rs.core.Response.Status.CREATED;
 import static jakarta.ws.rs.core.Response.Status.FORBIDDEN;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-import org.junit.jupiter.api.AfterAll;
-
+import static jakarta.ws.rs.core.Response.Status.OK;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
-
+import org.junit.jupiter.api.AfterAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
-import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
-
 public class ProvIT {
 
     private static boolean provEnabled = false;

From 75789e0f94d36fce1270b0714bd5e516f356d8ee Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 13 Nov 2023 19:06:26 -0500
Subject: [PATCH 721/922] current state of the flyway script (work in
 progress/likely to change) #8549

---
 .../V6.0.0.3__8549-collection-quotas.sql      | 70 +++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql

diff --git a/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql b/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
new file mode 100644
index 00000000000..f74d9bebe30
--- /dev/null
+++ b/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
@@ -0,0 +1,70 @@
+-- Storage size column added:
+ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS storagesize BIGINT;
+
+-- (work in progress! the table structure may change/the column may be moved out into
+-- its own table. but the mechanics of the recursion are working)
+
+-- The somewhat convoluted queries below populate the storage sizes for the entire
+-- DvObject tree, fast. It IS possible, to do it all with one recursive PostgresQL
+-- query, that will crawl the tree from the leaves (DataFiles) up and add up the
+-- sizes for all the Datasets/Collections above. Unfortunately, that takes some hours
+-- on a database the size of the one at IQSS. So what we are doing instead is compute
+-- the total sizes of all the *directly* linked objects, with 3 linear queries. This
+-- will correctly calculate the sizes of all the Datasets (since they can only
+-- contain DataFiles, directly, without any extra hierarchy possible) and those
+-- Collections that only contain Datasets; but not the sizes of Collections that
+-- have sub-collections. To take any sub-collections into account we are then running
+-- a recursive query - but then we only need to run it on the tree of Collections only,
+-- which should make it manageably fast on any real life instance. 
+
+UPDATE dvobject SET storagesize=0;
+-- For datafiles, the storage size = main file size by default:
+-- (we are excluding any harvested files)
+UPDATE dvobject SET storagesize=COALESCE(f.filesize,0) FROM datafile f, dataset d WHERE f.id = dvobject.id AND dvobject.owner_id = d.id AND d.harvestingclient_id IS null;
+-- ... but for ingested tabular files the size of the saved original needs to be added, since
+-- those also take space:
+-- (should be safe to assume that there are no *harvested ingested* files)
+UPDATE dvobject SET storagesize=dvobject.storagesize + COALESCE(datatable.originalFileSize,0) FROM datatable WHERE datatable.datafile_id = dvobject.id;
+-- Now we can calculate storage sizes of each individual dataset (a simple sum
+-- of the storage sizes of all the files in the dataset):
+-- (excluding the harvested datasets; this is less important, since there should be
+-- significantly fewer datasets than files, but might as well)
+UPDATE dvobject SET storagesize=o.combinedStorageSize
+FROM (SELECT datasetobject.id, SUM(fileobject.storagesize) AS combinedStorageSize
+FROM dvobject fileobject, dvobject datasetobject
+WHERE fileobject.owner_id = datasetobject.id
+GROUP BY datasetobject.id) o, dataset ds WHERE o.id = dvobject.id AND dvobject.dtype='Dataset' AND dvobject.id = ds.id AND ds.harvestingclient_id IS null;
+-- ... and then we can repeat the same for collections, by setting the storage size
+-- to the sum of the storage sizes of the datasets *directly* in each collection:
+-- (no attemp is made yet to recursively count the sizes all the chilld sub-collections)
+UPDATE dvobject SET storagesize=o.combinedStorageSize
+FROM (SELECT collectionobject.id, SUM(datasetobject.storagesize) AS combinedStorageSize
+FROM dvobject datasetobject, dvobject collectionobject
+WHERE datasetobject.owner_id = collectionobject.id
+AND datasetobject.storagesize IS NOT null 
+GROUP BY collectionobject.id) o WHERE o.id = dvobject.id AND dvobject.dtype='Dataverse';
+
+-- And now we will update the storage sizes of all the Collection ("Dataverse") objects
+-- that contain sub-collections, *recursively*, to add their sizes to the totals:
+WITH RECURSIVE treestorage (id, owner_id, storagesize, dtype) AS
+(
+    -- All dataverses:
+    SELECT id, owner_id, storagesize, dtype
+    FROM dvobject
+    WHERE dtype = 'Dataverse'
+
+    UNION
+
+    -- Recursive Member:
+    SELECT dvobject.id, treestorage.owner_id, dvobject.storagesize, treestorage.dtype
+    FROM treestorage, dvobject
+    WHERE treestorage.id = dvobject.owner_id
+    AND dvobject.dtype = 'Dataverse'
+)
+
+UPDATE dvobject SET storagesize=storagesize+(SELECT COALESCE(SUM(storagesize),0)
+FROM treestorage WHERE owner_id=dvobject.id)
+--FROM treestorage ts
+--WHERE ts.owner_id=dvobject.id
+WHERE dvobject.dtype = 'Dataverse'
+AND dvobject.id IN (SELECT owner_id FROM treestorage WHERE owner_id IS NOT null);

From c49036bf3d67d22cec384a8fe4f7cb23ed3d9a46 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 15 Nov 2023 12:06:43 +0000
Subject: [PATCH 722/922] Added: includeDeaccessioned support to
 getDatasetVersionCitation API endpoint

---
 .../harvard/iq/dataverse/api/Datasets.java    |  9 ++++++--
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 21 ++++++++++++++++++-
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  3 ++-
 3 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 292aba0cee3..68c618b0f1f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3995,9 +3995,14 @@ public Response getPrivateUrlDatasetVersionCitation(@PathParam("privateUrlToken"
     @GET
     @AuthRequired
     @Path("{id}/versions/{versionId}/citation")
-    public Response getDatasetVersionCitation(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+    public Response getDatasetVersionCitation(@Context ContainerRequestContext crc,
+                                              @PathParam("id") String datasetId,
+                                              @PathParam("versionId") String versionId,
+                                              @QueryParam("includeDeaccessioned") boolean includeDeaccessioned,
+                                              @Context UriInfo uriInfo,
+                                              @Context HttpHeaders headers) {
         return response(req -> ok(
-                getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers).getCitation(true, false)), getRequestUser(crc));
+                getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers, includeDeaccessioned).getCitation(true, false)), getRequestUser(crc));
     }
 
     @POST
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 56bf53c1c99..d20f1e8a58b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -3371,13 +3371,32 @@ public void getDatasetVersionCitation() {
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
 
-        Response getDatasetVersionCitationResponse = UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_DRAFT, apiToken);
+        Response getDatasetVersionCitationResponse = UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_DRAFT, false, apiToken);
         getDatasetVersionCitationResponse.prettyPrint();
 
         getDatasetVersionCitationResponse.then().assertThat()
                 .statusCode(OK.getStatusCode())
                 // We check that the returned message contains information expected for the citation string
                 .body("data.message", containsString("DRAFT VERSION"));
+
+        // Test Deaccessioned
+        Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken);
+        publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken);
+        publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        Response deaccessionDatasetResponse = UtilIT.deaccessionDataset(datasetId, DS_VERSION_LATEST_PUBLISHED, "Test deaccession reason.", null, apiToken);
+        deaccessionDatasetResponse.then().assertThat().statusCode(OK.getStatusCode());
+
+        // includeDeaccessioned false
+        Response getDatasetVersionCitationNotDeaccessioned = UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_LATEST_PUBLISHED, false, apiToken);
+        getDatasetVersionCitationNotDeaccessioned.then().assertThat().statusCode(NOT_FOUND.getStatusCode());
+
+        // includeDeaccessioned true
+        Response getDatasetVersionCitationDeaccessioned =  UtilIT.getDatasetVersionCitation(datasetId, DS_VERSION_LATEST_PUBLISHED, true, apiToken);
+        getDatasetVersionCitationDeaccessioned.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.message", containsString("DEACCESSIONED VERSION"));
     }
 
     @Test
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e3a7fd0cfc3..2336bf8beb8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3345,10 +3345,11 @@ static Response getPrivateUrlDatasetVersionCitation(String privateUrlToken) {
         return response;
     }
 
-    static Response getDatasetVersionCitation(Integer datasetId, String version, String apiToken) {
+    static Response getDatasetVersionCitation(Integer datasetId, String version, boolean includeDeaccessioned, String apiToken) {
         Response response = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json")
+                .queryParam("includeDeaccessioned", includeDeaccessioned)
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/citation");
         return response;
     }

From 75ff2fbad275a4543525ac0dc62f65d3eaa0e5c1 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 15 Nov 2023 12:10:14 +0000
Subject: [PATCH 723/922] Added: API docs for #10104

---
 doc/sphinx-guides/source/api/native-api.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 1992390410c..2e3a0b2af08 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2502,6 +2502,16 @@ Get Citation
 
   curl -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation?persistentId=$PERSISTENT_IDENTIFIER"
 
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+  curl -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation?persistentId=$PERSISTENT_IDENTIFIER&includeDeaccessioned=true"
+
 Get Citation by Private URL Token
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

From be631af6e5fd5dd181aebdb0ee8a2dd1da3ff789 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Wed, 15 Nov 2023 12:12:31 +0000
Subject: [PATCH 724/922] Added: release notes for #10104

---
 doc/release-notes/10104-dataset-citation-deaccessioned.md | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 doc/release-notes/10104-dataset-citation-deaccessioned.md

diff --git a/doc/release-notes/10104-dataset-citation-deaccessioned.md b/doc/release-notes/10104-dataset-citation-deaccessioned.md
new file mode 100644
index 00000000000..0ba06d729c4
--- /dev/null
+++ b/doc/release-notes/10104-dataset-citation-deaccessioned.md
@@ -0,0 +1 @@
+The getDatasetVersionCitation (/api/datasets/{id}/versions/{versionId}/citation) endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation.

From 2fb81f6b5e1a5c735b937600b0dd74ee47d236a1 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 15 Nov 2023 10:01:52 -0500
Subject: [PATCH 725/922] altering circuit breakers for qa

---
 conf/solr/9.3.0/solrconfig.xml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/conf/solr/9.3.0/solrconfig.xml b/conf/solr/9.3.0/solrconfig.xml
index b89315cdaa9..9705faa7009 100644
--- a/conf/solr/9.3.0/solrconfig.xml
+++ b/conf/solr/9.3.0/solrconfig.xml
@@ -588,10 +588,10 @@
      check for "Circuit Breakers tripped" in logs and the corresponding error message should tell
      you what transpired (if the failure was caused by tripped circuit breakers).
     -->
-    <!--
+
     <str name="memEnabled">true</str>
-    <str name="memThreshold">75</str>
-    -->
+    <str name="memThreshold">5</str>
+
 
     <!-- CPU Circuit Breaker Configuration
 
@@ -599,10 +599,10 @@
      whether the circuit breaker is enabled and the average load over the last minute at which the
      circuit breaker should start rejecting queries.
     -->
-    <!--
+
     <str name="cpuEnabled">true</str>
-    <str name="cpuThreshold">75</str>
-    -->
+    <str name="cpuThreshold">5</str>
+
   </circuitBreaker>
 
   <!-- Request Dispatcher

From 072cbeaa475eb734596df14c104736f7cc461e1d Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 15 Nov 2023 10:23:23 -0500
Subject: [PATCH 726/922] revert circuit breaker changes

---
 conf/solr/9.3.0/solrconfig.xml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/conf/solr/9.3.0/solrconfig.xml b/conf/solr/9.3.0/solrconfig.xml
index 9705faa7009..36ed4f23390 100644
--- a/conf/solr/9.3.0/solrconfig.xml
+++ b/conf/solr/9.3.0/solrconfig.xml
@@ -588,10 +588,11 @@
      check for "Circuit Breakers tripped" in logs and the corresponding error message should tell
      you what transpired (if the failure was caused by tripped circuit breakers).
     -->
-
+    
+    <!--
     <str name="memEnabled">true</str>
-    <str name="memThreshold">5</str>
-
+    <str name="memThreshold">75</str>
+    -->
 
     <!-- CPU Circuit Breaker Configuration
 
@@ -599,9 +600,11 @@
      whether the circuit breaker is enabled and the average load over the last minute at which the
      circuit breaker should start rejecting queries.
     -->
-
+    
+    <!--
     <str name="cpuEnabled">true</str>
-    <str name="cpuThreshold">5</str>
+    <str name="cpuThreshold">75</str>
+    -->
 
   </circuitBreaker>
 

From a376b4e3f4bacc8dc651b7048d9a323535dc92f7 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Fri, 17 Nov 2023 10:01:33 -0500
Subject: [PATCH 727/922] Add condition for 401 when a invalid key is provided
 and create changelog on API Guide

---
 doc/sphinx-guides/source/api/changelog.rst          | 13 +++++++++++++
 doc/sphinx-guides/source/api/index.rst              |  1 +
 .../java/edu/harvard/iq/dataverse/api/AccessIT.java | 11 ++++++-----
 3 files changed, 20 insertions(+), 5 deletions(-)
 create mode 100644 doc/sphinx-guides/source/api/changelog.rst

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
new file mode 100644
index 00000000000..b78d268db33
--- /dev/null
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -0,0 +1,13 @@
+API Changelog 
+=============
+
+.. contents:: |toctitle|
+    :local:
+    :depth: 1
+
+6.0.0
+-----
+
+Changes
+~~~~~~~
+    - **api/access/datafile**: When a null or invalid API Key is provided to download a public with this API call, it will result on a ``401`` error response. 
diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst
index c9e79098546..dd195aa9d62 100755
--- a/doc/sphinx-guides/source/api/index.rst
+++ b/doc/sphinx-guides/source/api/index.rst
@@ -24,3 +24,4 @@ API Guide
    linkeddatanotification
    apps
    faq
+   changelog
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
index 42e21e53101..d08f916243f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java
@@ -198,6 +198,8 @@ public void testDownloadSingleFile() {
         //Not logged in non-restricted
         Response anonDownloadOriginal = UtilIT.downloadFileOriginal(tabFile1Id);
         Response anonDownloadConverted = UtilIT.downloadFile(tabFile1Id);
+        Response anonDownloadConvertedNullKey = UtilIT.downloadFile(tabFile1Id, null);
+
         // ... and download the same tabular data file, but without the variable name header added:
         Response anonDownloadTabularNoHeader = UtilIT.downloadTabularFileNoVarHeader(tabFile1Id);
         // ... and download the same tabular file, this time requesting the "format=tab" explicitly:
@@ -206,6 +208,8 @@ public void testDownloadSingleFile() {
         assertEquals(OK.getStatusCode(), anonDownloadConverted.getStatusCode());
         assertEquals(OK.getStatusCode(), anonDownloadTabularNoHeader.getStatusCode());
         assertEquals(OK.getStatusCode(), anonDownloadTabularWithFormatName.getStatusCode());
+        assertEquals(UNAUTHORIZED.getStatusCode(), anonDownloadConvertedNullKey.getStatusCode());
+        
         int origSizeAnon = anonDownloadOriginal.getBody().asByteArray().length;
         int convertSizeAnon = anonDownloadConverted.getBody().asByteArray().length;
         int tabularSizeNoVarHeader = anonDownloadTabularNoHeader.getBody().asByteArray().length;
@@ -423,10 +427,7 @@ private HashMap<String,ByteArrayOutputStream> readZipResponse(InputStream iStrea
                 }
 
                 String name = entry.getName(); 
-//                String s = String.format("Entry: %s len %d added %TD",
-//                                entry.getName(), entry.getSize(),
-//                                new Date(entry.getTime()));
-//                System.out.println(s);
+
 
                 // Once we get the entry from the zStream, the zStream is
                 // positioned read to read the raw data, and we keep
@@ -466,7 +467,7 @@ private HashMap<String,ByteArrayOutputStream> readZipResponse(InputStream iStrea
     
     @Test
     public void testRequestAccess() throws InterruptedException {
-
+    
         String pathToJsonFile = "scripts/api/data/dataset-create-new.json";
         Response createDatasetResponse = UtilIT.createDatasetViaNativeApi(dataverseAlias, pathToJsonFile, apiToken);
         createDatasetResponse.prettyPrint();

From 63725d75c115352ff9d0bb94f2e5b6b4d7ca5d05 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 17 Nov 2023 11:07:17 -0500
Subject: [PATCH 728/922] remove cruft: mdc logs #9115

---
 mdc-logs/raw-mdc-2019-01-07.log | 6 ------
 1 file changed, 6 deletions(-)
 delete mode 100644 mdc-logs/raw-mdc-2019-01-07.log

diff --git a/mdc-logs/raw-mdc-2019-01-07.log b/mdc-logs/raw-mdc-2019-01-07.log
deleted file mode 100644
index d7a6386160e..00000000000
--- a/mdc-logs/raw-mdc-2019-01-07.log
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fields: event_time	client_ip	session_cookie_id	user_cookie_id	user_id	request_url	identifier	filename	size	user-agent	title	publisher	publisher_id	authors	publication_date	version	other_id	target_url	publication_year
-2019-01-07T15:14:51-0500	0:0:0:0:0:0:0:1	9f4209d3c177d3cb77f4d06cf3ba	-	:guest	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	doi:10.5072/FK2/XTT5BV	-	-	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Dataset One	-	1	Smith, Robert| Kew, Susie	2019-01-07T18:20:54Z	1	-	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	2019
-2019-01-07T15:15:15-0500	0:0:0:0:0:0:0:1	9f4209d3c177d3cb77f4d06cf3ba	-	:guest	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	doi:10.5072/FK2/XTT5BV	-	-	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Dataset One	-	1	Smith, Robert| Kew, Susie	2019-01-07T18:20:54Z	1	-	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	2019
-2019-01-07T15:16:04-0500	0:0:0:0:0:0:0:1	9f4209d3c177d3cb77f4d06cf3ba	-	:guest	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	doi:10.5072/FK2/XTT5BV	-	-	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Dataset One	-	1	Smith, Robert| Kew, Susie	2019-01-07T18:20:54Z	1	-	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	2019
-2019-01-07T15:16:14-0500	0:0:0:0:0:0:0:1	9f4209d3c177d3cb77f4d06cf3ba	-	:guest	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	doi:10.5072/FK2/XTT5BV	168298bae7c-2c5bbc1a9c8c	1	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Dataset One	-	1	Smith, Robert| Kew, Susie	2019-01-07T18:20:54Z	1	-	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	2019
-2019-01-07T15:16:19-0500	0:0:0:0:0:0:0:1	9f4209d3c177d3cb77f4d06cf3ba	-	:guest	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	doi:10.5072/FK2/XTT5BV	168298bb8ce-337d8df49763	4026	Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36	Dataset One	-	1	Smith, Robert| Kew, Susie	2019-01-07T18:20:54Z	1	-	http://localhost:8080/dataset.xhtml?persistentId=doi:10.5072/FK2/XTT5BV	2019

From 2433114ec7b8430753bc730056a07e24ac0bb5d3 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 17 Nov 2023 11:20:03 -0500
Subject: [PATCH 729/922] fix bullet #10060 #10070

---
 doc/sphinx-guides/source/api/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index b78d268db33..a1cffd84f33 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -10,4 +10,4 @@ API Changelog
 
 Changes
 ~~~~~~~
-    - **api/access/datafile**: When a null or invalid API Key is provided to download a public with this API call, it will result on a ``401`` error response. 
+- **api/access/datafile**: When a null or invalid API Key is provided to download a public with this API call, it will result on a ``401`` error response. 

From e0350e735551270f9bd23bfa226b6946282df467 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Fri, 17 Nov 2023 11:38:53 -0500
Subject: [PATCH 730/922] Change 6.0.0 to 6.0

---
 doc/sphinx-guides/source/api/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index a1cffd84f33..086ff4a20e5 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -5,7 +5,7 @@ API Changelog
     :local:
     :depth: 1
 
-6.0.0
+6.0
 -----
 
 Changes

From 437e3b94edf89a2245310709c07d8238c0df4235 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva
 <142103991+jp-tosca@users.noreply.github.com>
Date: Fri, 17 Nov 2023 11:42:17 -0500
Subject: [PATCH 731/922] Update doc/sphinx-guides/source/api/changelog.rst

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/api/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index 086ff4a20e5..2698ba3debf 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -10,4 +10,4 @@ API Changelog
 
 Changes
 ~~~~~~~
-- **api/access/datafile**: When a null or invalid API Key is provided to download a public with this API call, it will result on a ``401`` error response. 
+- **/api/access/datafile**: When a null or invalid API Key is provided to download a public with this API call, it will result on a ``401`` error response. 

From 640f69e39f71244b9ba1d7f534180a6b4c8b58cc Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 17 Nov 2023 13:19:14 -0500
Subject: [PATCH 732/922] add release note for API changelog #10060

---
 doc/release-notes/10060-api-changelog.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 doc/release-notes/10060-api-changelog.md

diff --git a/doc/release-notes/10060-api-changelog.md b/doc/release-notes/10060-api-changelog.md
new file mode 100644
index 00000000000..56ac96e3564
--- /dev/null
+++ b/doc/release-notes/10060-api-changelog.md
@@ -0,0 +1,3 @@
+We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
+
+See also #10060.

From 83a66aac65db2f7634b3917d332b0e4253be3c84 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva
 <142103991+jp-tosca@users.noreply.github.com>
Date: Fri, 17 Nov 2023 14:55:58 -0500
Subject: [PATCH 733/922] Update doc/sphinx-guides/source/api/changelog.rst

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/sphinx-guides/source/api/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index 2698ba3debf..f518a9b542d 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -10,4 +10,4 @@ API Changelog
 
 Changes
 ~~~~~~~
-- **/api/access/datafile**: When a null or invalid API Key is provided to download a public with this API call, it will result on a ``401`` error response. 
+- **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed to happy (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch.

From 70edaa789e84c99b110036c232155337afb5c459 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Fri, 17 Nov 2023 15:02:32 -0500
Subject: [PATCH 734/922] Remove "to happy "

---
 doc/sphinx-guides/source/api/changelog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index f518a9b542d..d6742252d27 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -10,4 +10,4 @@ API Changelog
 
 Changes
 ~~~~~~~
-- **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed to happy (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch.
+- **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch.

From 73593acb1bcdb9ba1d62e47310753e905b2546dd Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Fri, 17 Nov 2023 15:17:28 -0500
Subject: [PATCH 735/922] #9464 query by dvo. update IT

---
 .../dataverse/metrics/MetricsServiceBean.java | 33 ++++++++++---------
 .../harvard/iq/dataverse/api/MetricsIT.java   | 14 +++++---
 2 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index 79369207963..832dda5ced9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -138,8 +138,8 @@ public JsonArray getDatasetsTimeSeries(UriInfo uriInfo, String dataLocation, Dat
                 + "from datasetversion\n"
                 + "where versionstate='RELEASED' \n"
                 + (((d == null)&&(DATA_LOCATION_ALL.equals(dataLocation))) ? "" : "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id=dvobject.id\n")
-                + ((DATA_LOCATION_LOCAL.equals(dataLocation)) ? "and dataset.harvestingclient_id IS NULL and publicationdate is not null\n " : "")
-                + ((DATA_LOCATION_REMOTE.equals(dataLocation)) ? "and dataset.harvestingclient_id IS NOT NULL\n "  : "")
+                + ((DATA_LOCATION_LOCAL.equals(dataLocation)) ? "and dvobject.harvestingclient_id IS NULL and publicationdate is not null\n " : "")
+                + ((DATA_LOCATION_REMOTE.equals(dataLocation)) ? "and dvobject.harvestingclient_id IS NOT NULL\n "  : "")
                 + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n ")
                 + (((d == null)&&(DATA_LOCATION_ALL.equals(dataLocation))) ? "" : ")\n")
                 + "group by dataset_id) as subq group by subq.date order by date;"
@@ -156,11 +156,11 @@ public JsonArray getDatasetsTimeSeries(UriInfo uriInfo, String dataLocation, Dat
      * @param d
      */
     public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
-        String dataLocationLine = "(date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM') and dataset.harvestingclient_id IS NULL)\n";
+        String dataLocationLine = "(date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM') and dvobject.harvestingclient_id IS NULL)\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
             //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
-            String harvestBaseLine = "(date_trunc('month', createtime) <=  to_date('" + yyyymm + "','YYYY-MM') and dataset.harvestingclient_id IS NOT NULL)\n";
+            String harvestBaseLine = "(date_trunc('month', createtime) <=  to_date('" + yyyymm + "','YYYY-MM') and dvobject.harvestingclient_id IS NOT NULL)\n";
             if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
                 dataLocationLine = harvestBaseLine; // replace
             } else if (DATA_LOCATION_ALL.equals(dataLocation)) {
@@ -189,7 +189,7 @@ public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
                         + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
                         + "from datasetversion\n"
                         + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                        + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
+                        + "join dvobject on dvobject.id = dataset.id\n"
                         + "where versionstate='RELEASED' \n"
                         + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n ")
                         + "and \n"
@@ -212,8 +212,9 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
                 "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n" +
                 "       from datasetversion\n" +
                 "       join dataset on dataset.id = datasetversion.dataset_id\n" +
+                "       join dvobject on dataset.id = dvobject.id\n" +
                 "       where versionstate='RELEASED'\n" +
-                "       	     and dataset.harvestingclient_id is null\n" +
+                "       	     and dvobject.harvestingclient_id is null\n" +
                 "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n" +
                 "       group by dataset_id\n" +
                 "))\n";
@@ -225,7 +226,7 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
             // so the query is simpler:
             String harvestOriginClause = "(\n" +
                     "   datasetversion.dataset_id = dataset.id\n" +
-                    "   AND dataset.harvestingclient_id IS NOT null \n" +
+                    "   AND dvobject.harvestingclient_id IS NOT null \n" +
                     "   AND date_trunc('month', datasetversion.createtime) <=  to_date('" + yyyymm + "','YYYY-MM')\n" +
                     ")\n";
 
@@ -244,7 +245,7 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
                 + "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n"
                 + "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n"
                 + "JOIN dataset ON dataset.id = datasetversion.dataset_id\n"
-                + ((d == null) ? "" : "JOIN dvobject ON dvobject.id = dataset.id\n")
+                + "JOIN dvobject ON dvobject.id = dataset.id\n"
                 + "WHERE\n"
                 + originClause
                 + "AND datasetfieldtype.name = 'subject'\n"
@@ -258,11 +259,11 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
     }
 
     public long datasetsPastDays(int days, String dataLocation, Dataverse d) {
-        String dataLocationLine = "(releasetime > current_date - interval '" + days + "' day and dataset.harvestingclient_id IS NULL)\n";
+        String dataLocationLine = "(releasetime > current_date - interval '" + days + "' day and dvobject.harvestingclient_id IS NULL)\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
             //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
-            String harvestBaseLine = "(createtime > current_date - interval '" + days + "' day and dataset.harvestingclient_id IS NOT NULL)\n";
+            String harvestBaseLine = "(createtime > current_date - interval '" + days + "' day and dvobject.harvestingclient_id IS NOT NULL)\n";
             if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
                 dataLocationLine = harvestBaseLine; // replace
             } else if (DATA_LOCATION_ALL.equals(dataLocation)) {
@@ -276,7 +277,7 @@ public long datasetsPastDays(int days, String dataLocation, Dataverse d) {
                         + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max\n"
                         + "from datasetversion\n"
                         + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                        + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
+                        + "join dvobject on dvobject.id = dataset.id\n"
                         + "where versionstate='RELEASED' \n"
                         + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                         + "and \n"
@@ -304,7 +305,7 @@ public JsonArray filesTimeSeries(Dataverse d) {
                         + "where datasetversion.id=filemetadata.datasetversion_id\n"
                         + "and versionstate='RELEASED' \n"
                         + "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id=dvobject.id\n"
-                        + "and dataset.harvestingclient_id IS NULL and publicationdate is not null\n "
+                        + "and dvobject.harvestingclient_id IS NULL and publicationdate is not null\n "
                         + ((d == null) ? ")" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + "))\n ")
                         + "group by filemetadata.id) as subq group by subq.date order by date;");
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -327,11 +328,11 @@ public long filesToMonth(String yyyymm, Dataverse d) {
                 + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
+                +  "join dvobject on dvobject.id = dataset.id\n"
                 + "where versionstate='RELEASED'\n"
                 + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                 + "and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
-                + "and dataset.harvestingclient_id is null\n"
+                + "and dvobject.harvestingclient_id is null\n"
                 + "group by dataset_id \n"
                 + ");"
         );
@@ -350,11 +351,11 @@ public long filesPastDays(int days, Dataverse d) {
                 + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
+                + "join dvobject on dvobject.id = dataset.id\n"
                 + "where versionstate='RELEASED'\n"
                 + "and releasetime > current_date - interval '" + days + "' day\n"
                 + ((d == null) ? "" : "AND dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
-                + "and dataset.harvestingclient_id is null\n"
+                + "and dvobject.harvestingclient_id is null\n"
                 + "group by dataset_id \n"
                 + ");"
         );
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java
index e3328eefb4a..b961a86dc0b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java
@@ -30,7 +30,7 @@ public static void cleanUpClass() {
 
     @Test
     public void testGetDataversesToMonth() {
-        String yyyymm = "2018-04";
+        String yyyymm = "2023-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsDataversesToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -54,7 +54,7 @@ public void testGetDataversesToMonth() {
 
     @Test
     public void testGetDatasetsToMonth() {
-        String yyyymm = "2018-04";
+        String yyyymm = "2023-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsDatasetsToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -77,7 +77,7 @@ public void testGetDatasetsToMonth() {
 
     @Test
     public void testGetFilesToMonth() {
-        String yyyymm = "2018-04";
+        String yyyymm = "2023-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsFilesToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -100,7 +100,7 @@ public void testGetFilesToMonth() {
 
     @Test
     public void testGetDownloadsToMonth() {
-        String yyyymm = "2018-04";
+        String yyyymm = "2023-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsDownloadsToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -283,6 +283,12 @@ public void testGetDatasetsBySubject() {
         response = UtilIT.metricsDatasetsBySubject("dataLocation=local");
         response.then().assertThat()
                 .statusCode(OK.getStatusCode());
+        
+        //Test ok when passing remote
+        response = UtilIT.metricsDatasetsBySubject("dataLocation=remote");
+        response.prettyPrint();
+        response.then().assertThat()
+                .statusCode(OK.getStatusCode());
     }
 
     @Test

From d0fc9affdf52dfd60461520adb20a6c7d30e7d6b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 17 Nov 2023 15:31:50 -0500
Subject: [PATCH 736/922] refactor to avoid overloaded methods in constructors

---
 .../AbstractRemoteOverlayAccessIO.java        | 335 ++++++++++++++++++
 .../dataaccess/GlobusAccessibleStore.java     |   4 +-
 .../dataaccess/GlobusOverlayAccessIO.java     |  51 ++-
 .../dataaccess/RemoteOverlayAccessIO.java     | 315 +---------------
 .../dataaccess/RemoteOverlayAccessIOTest.java |   1 -
 5 files changed, 390 insertions(+), 316 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
new file mode 100644
index 00000000000..8adaf746210
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
@@ -0,0 +1,335 @@
+package edu.harvard.iq.dataverse.dataaccess;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.channels.Channel;
+import java.nio.file.Path;
+import java.security.KeyManagementException;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+
+import javax.net.ssl.SSLContext;
+
+import org.apache.http.Header;
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.protocol.HTTP;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.apache.http.util.EntityUtils;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DvObject;
+
+public abstract class AbstractRemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
+
+    protected static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
+    protected static final String REFERENCE_ENDPOINTS_WITH_BASEPATHS = "reference-endpoints-with-basepaths";
+    static final String BASE_STORE = "base-store";
+    protected static final String SECRET_KEY = "secret-key";
+    static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
+    protected static final String REMOTE_STORE_NAME = "remote-store-name";
+    protected static final String REMOTE_STORE_URL = "remote-store-url";
+    protected StorageIO<DvObject> baseStore = null;
+    protected String path = null;
+    protected PoolingHttpClientConnectionManager cm = null;
+    CloseableHttpClient httpclient = null;
+    protected static HttpClientContext localContext = HttpClientContext.create();
+
+    protected int timeout = 1200;
+    protected RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
+                .setConnectionRequestTimeout(timeout * 1000).setSocketTimeout(timeout * 1000)
+                .setCookieSpec(CookieSpecs.STANDARD).setExpectContinueEnabled(true).build();
+    protected static boolean trustCerts = false;
+    protected int httpConcurrency = 4;
+
+    public static String getBaseStoreIdFor(String driverId) {
+        return getConfigParamForDriver(driverId, BASE_STORE);
+    }
+
+    public AbstractRemoteOverlayAccessIO() {
+        super();
+    }
+
+    public AbstractRemoteOverlayAccessIO(String storageLocation, String driverId) {
+        super(storageLocation, driverId);
+    }
+
+    public AbstractRemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) {
+        super(dvObject, req, driverId);
+    }
+
+    @Override
+    public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException {
+        return baseStore.openAuxChannel(auxItemTag, options);
+    }
+
+    @Override
+    public boolean isAuxObjectCached(String auxItemTag) throws IOException {
+        return baseStore.isAuxObjectCached(auxItemTag);
+    }
+
+    @Override
+    public long getAuxObjectSize(String auxItemTag) throws IOException {
+        return baseStore.getAuxObjectSize(auxItemTag);
+    }
+
+    @Override
+    public Path getAuxObjectAsPath(String auxItemTag) throws IOException {
+        return baseStore.getAuxObjectAsPath(auxItemTag);
+    }
+
+    @Override
+    public void backupAsAux(String auxItemTag) throws IOException {
+        baseStore.backupAsAux(auxItemTag);
+    }
+
+    @Override
+    public void revertBackupAsAux(String auxItemTag) throws IOException {
+        baseStore.revertBackupAsAux(auxItemTag);
+    }
+
+    @Override
+    public void savePathAsAux(Path fileSystemPath, String auxItemTag) throws IOException {
+        baseStore.savePathAsAux(fileSystemPath, auxItemTag);
+    }
+
+    @Override
+    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Long filesize) throws IOException {
+        baseStore.saveInputStreamAsAux(inputStream, auxItemTag, filesize);
+    }
+
+    /**
+     * @param inputStream InputStream we want to save
+     * @param auxItemTag  String representing this Auxiliary type ("extension")
+     * @throws IOException if anything goes wrong.
+     */
+    @Override
+    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException {
+        baseStore.saveInputStreamAsAux(inputStream, auxItemTag);
+    }
+
+    @Override
+    public List<String> listAuxObjects() throws IOException {
+        return baseStore.listAuxObjects();
+    }
+
+    @Override
+    public void deleteAuxObject(String auxItemTag) throws IOException {
+        baseStore.deleteAuxObject(auxItemTag);
+    }
+
+    @Override
+    public void deleteAllAuxObjects() throws IOException {
+        baseStore.deleteAllAuxObjects();
+    }
+
+    @Override
+    public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException {
+        return baseStore.getAuxFileAsInputStream(auxItemTag);
+    }
+
+    protected int getUrlExpirationMinutes() {
+        String optionValue = getConfigParam(URL_EXPIRATION_MINUTES);
+        if (optionValue != null) {
+            Integer num;
+            try {
+                num = Integer.parseInt(optionValue);
+            } catch (NumberFormatException ex) {
+                num = null;
+            }
+            if (num != null) {
+                return num;
+            }
+        }
+        return 60;
+    }
+
+    public CloseableHttpClient getSharedHttpClient() {
+        if (httpclient == null) {
+            try {
+                initHttpPool();
+                httpclient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(config).build();
+    
+            } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException ex) {
+                logger.warning(ex.getMessage());
+            }
+        }
+        return httpclient;
+    }
+
+    private void initHttpPool() throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException {
+        if (trustCerts) {
+            // use the TrustSelfSignedStrategy to allow Self Signed Certificates
+            SSLContext sslContext;
+            SSLConnectionSocketFactory connectionFactory;
+    
+            sslContext = SSLContextBuilder.create().loadTrustMaterial(new TrustAllStrategy()).build();
+            // create an SSL Socket Factory to use the SSLContext with the trust self signed
+            // certificate strategy
+            // and allow all hosts verifier.
+            connectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
+    
+            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
+                    .register("https", connectionFactory).build();
+            cm = new PoolingHttpClientConnectionManager(registry);
+        } else {
+            cm = new PoolingHttpClientConnectionManager();
+        }
+        cm.setDefaultMaxPerRoute(httpConcurrency);
+        cm.setMaxTotal(httpConcurrency > 20 ? httpConcurrency : 20);
+    }
+
+    @Override
+    abstract public long retrieveSizeFromMedia();
+    
+    @Override
+    public boolean exists() {
+        logger.fine("Exists called");
+        return (retrieveSizeFromMedia() != -1);
+    }
+
+    @Override
+    public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOException {
+        return baseStore.cleanUp(filter, dryRun);
+    }
+    
+    @Override
+    public String getStorageLocation() throws IOException {
+        String fullStorageLocation = dvObject.getStorageIdentifier();
+        logger.fine("storageidentifier: " + fullStorageLocation);
+        int driverIndex = fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR);
+        if (driverIndex >= 0) {
+            fullStorageLocation = fullStorageLocation
+                    .substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        }
+        if (this.getDvObject() instanceof Dataset) {
+            throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
+        } else if (this.getDvObject() instanceof DataFile) {
+            fullStorageLocation = StorageIO.getDriverPrefix(this.driverId) + fullStorageLocation;
+        } else if (dvObject instanceof Dataverse) {
+            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
+        }
+        logger.fine("fullStorageLocation: " + fullStorageLocation);
+        return fullStorageLocation;
+    }
+    protected void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
+
+        if (baseStore == null) {
+            String baseDriverId = getBaseStoreIdFor(driverId);
+            String fullStorageLocation = null;
+            String baseDriverType = getConfigParamForDriver(baseDriverId, StorageIO.TYPE,
+                    DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+
+            if (dvObject instanceof Dataset) {
+                baseStore = DataAccess.getStorageIO(dvObject, req, baseDriverId);
+            } else {
+                if (this.getDvObject() != null) {
+                    fullStorageLocation = getStoragePath();
+
+                    // S3 expects <id>://<bucketname>/<key>
+                    switch (baseDriverType) {
+                    case DataAccess.S3:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + getConfigParamForDriver(baseDriverId, S3AccessIO.BUCKET_NAME) + "/"
+                                + fullStorageLocation;
+                        break;
+                    case DataAccess.FILE:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + getConfigParamForDriver(baseDriverId, FileAccessIO.DIRECTORY, "/tmp/files")
+                                + "/" + fullStorageLocation;
+                        break;
+                    default:
+                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
+                                + getConfigParamForDriver(baseDriverId, StorageIO.TYPE));
+                        throw new IOException("Not supported");
+                    }
+
+                } else if (storageLocation != null) {
+                    // <remoteDriverId>://<baseStorageIdentifier>//<baseUrlPath>
+                    // remoteDriverId:// is removed if coming through directStorageIO
+                    int index = storageLocation.indexOf(DataAccess.SEPARATOR);
+                    if (index > 0) {
+                        storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
+                    }
+                    // The base store needs the baseStoreIdentifier and not the relative URL (if it exists)
+                    int endOfId = storageLocation.indexOf("//");
+                    fullStorageLocation = (endOfId>-1) ? storageLocation.substring(0, endOfId) : storageLocation;
+
+                    switch (baseDriverType) {
+                    case DataAccess.S3:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + getConfigParamForDriver(baseDriverId, S3AccessIO.BUCKET_NAME) + "/"
+                                + fullStorageLocation;
+                        break;
+                    case DataAccess.FILE:
+                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
+                                + getConfigParamForDriver(baseDriverId, FileAccessIO.DIRECTORY, "/tmp/files")
+                                + "/" + fullStorageLocation;
+                        break;
+                    default:
+                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
+                                + getConfigParamForDriver(baseDriverId, StorageIO.TYPE));
+                        throw new IOException("Not supported");
+                    }
+                }
+                baseStore = DataAccess.getDirectStorageIO(fullStorageLocation);
+            }
+            if (baseDriverType.contentEquals(DataAccess.S3)) {
+                ((S3AccessIO<?>) baseStore).setMainDriver(false);
+            }
+        }
+        remoteStoreName = getConfigParam(REMOTE_STORE_NAME);
+        try {
+            remoteStoreUrl = new URL(getConfigParam(REMOTE_STORE_URL));
+        } catch (MalformedURLException mfue) {
+            logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
+        }
+    }
+
+    protected String getStoragePath() throws IOException {
+        String fullStoragePath = dvObject.getStorageIdentifier();
+        logger.fine("storageidentifier: " + fullStoragePath);
+        int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
+        if (driverIndex >= 0) {
+            fullStoragePath = fullStoragePath
+                    .substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
+        }
+        int suffixIndex = fullStoragePath.indexOf("//");
+        if (suffixIndex >= 0) {
+            fullStoragePath = fullStoragePath.substring(0, suffixIndex);
+        }
+        if (getDvObject() instanceof Dataset) {
+            fullStoragePath = getDataset().getAuthorityForFileStorage() + "/"
+                    + getDataset().getIdentifierForFileStorage() + "/" + fullStoragePath;
+        } else if (getDvObject() instanceof DataFile) {
+            fullStoragePath = getDataFile().getOwner().getAuthorityForFileStorage() + "/"
+                    + getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath;
+        } else if (dvObject instanceof Dataverse) {
+            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
+        }
+        logger.fine("fullStoragePath: " + fullStoragePath);
+        return fullStoragePath;
+    }
+
+
+
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index afc7556481a..ce75395c883 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -28,7 +28,7 @@ public static String getTransferPath(String driverId) {
     }
 
     public static JsonArray getReferenceEndpointsWithPaths(String driverId) {
-        String[] endpoints = StorageIO.getConfigParamForDriver(driverId, RemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*");
+        String[] endpoints = StorageIO.getConfigParamForDriver(driverId, AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS).split("\\s*,\\s*");
         JsonArrayBuilder builder = Json.createArrayBuilder();
         for(int i=0;i<endpoints.length;i++) {
             builder.add(endpoints[i]);
@@ -44,7 +44,7 @@ public static boolean acceptsGlobusTransfers(String storeId) {
     }
 
     public static boolean allowsGlobusReferences(String storeId) {
-        if(StorageIO.getConfigParamForDriver(storeId, RemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS) != null) {
+        if(StorageIO.getConfigParamForDriver(storeId, AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS) != null) {
             return true;
         }
         return false;
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index f42f5443108..a37cd23ad2b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -13,6 +13,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.logging.Logger;
@@ -46,7 +47,7 @@
  * reference endpoints separated by a comma
  * 
  */
-public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAccessIO<T> implements GlobusAccessibleStore {
+public class GlobusOverlayAccessIO<T extends DvObject> extends AbstractRemoteOverlayAccessIO<T> implements GlobusAccessibleStore {
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIO");
 
     /*
@@ -67,11 +68,19 @@ public class GlobusOverlayAccessIO<T extends DvObject> extends RemoteOverlayAcce
 
     public GlobusOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
+        configureGlobusEndpoints();
+        configureStores(req, driverId, null);
+        logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
+        path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
+        validatePath(path);
+
+        logger.fine("Relative path: " + path);
     }
 
 
     public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOException {
         this.driverId = driverId;
+        configureGlobusEndpoints();
         configureStores(null, driverId, storageLocation);
         if (isManaged()) {
             String[] parts = DataAccess.getDriverIdAndStorageLocation(storageLocation);
@@ -83,6 +92,7 @@ public GlobusOverlayAccessIO(String storageLocation, String driverId) throws IOE
             logger.fine("Referenced path: " + path);
         }
     }
+    
     private boolean isManaged() {
         if(dataverseManaged==null) {
             dataverseManaged = GlobusAccessibleStore.isDataverseManaged(this.driverId);
@@ -146,7 +156,6 @@ private static String findMatchingEndpoint(String path, String[] allowedEndpoint
         return null;
     }
 
-    @Override
     protected void validatePath(String relPath) throws IOException {
         if (isManaged()) {
             if (!usesStandardNamePattern(relPath)) {
@@ -363,8 +372,7 @@ public String getStorageLocation() throws IOException {
      * the derived GlobusOverlayAccessIO can support multiple endpoints.
      * @throws IOException
      */
-    @Override
-    protected void configureEndpoints() throws IOException {
+    protected void configureGlobusEndpoints() throws IOException {
         allowedEndpoints = getAllowedEndpoints(this.driverId);
         logger.info("Set allowed endpoints: " + Arrays.toString(allowedEndpoints));
     }
@@ -435,5 +443,40 @@ public static void main(String[] args) {
         }
 
     }
+
+
+    @Override
+    public void open(DataAccessOption... option) throws IOException {
+        // TODO Auto-generated method stub
+        
+    }
+
+
+    @Override
+    public Path getFileSystemPath() throws IOException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+
+    @Override
+    public void savePath(Path fileSystemPath) throws IOException {
+        // TODO Auto-generated method stub
+        
+    }
+
+
+    @Override
+    public void saveInputStream(InputStream inputStream) throws IOException {
+        // TODO Auto-generated method stub
+        
+    }
+
+
+    @Override
+    public void saveInputStream(InputStream inputStream, Long filesize) throws IOException {
+        // TODO Auto-generated method stub
+        
+    }
     
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
index 5463254140d..1616bfabf96 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIO.java
@@ -11,45 +11,23 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.channels.Channel;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.nio.file.Path;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
 import java.util.List;
-import java.util.function.Predicate;
-import java.util.logging.Logger;
 
 import org.apache.http.Header;
-import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.config.Registry;
-import org.apache.http.config.RegistryBuilder;
-import org.apache.http.conn.socket.ConnectionSocketFactory;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.TrustAllStrategy;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.protocol.HTTP;
-import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.util.EntityUtils;
 
-import javax.net.ssl.SSLContext;
-
 /**
  * @author qqmyers
  */
@@ -61,40 +39,20 @@
  * 
  * baseUrl: http(s)://<host(:port)/basePath>
  */
-public class RemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
-
-    private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
+public class RemoteOverlayAccessIO<T extends DvObject> extends AbstractRemoteOverlayAccessIO<T> {
 
     // A single baseUrl of the form http(s)://<host(:port)/basePath> where this store can reference data
     static final String BASE_URL = "base-url";
-    // Multiple endpoints where data can be referenced from. Multiple endpoints are separated by a comma. Multiple endpoints are only supported by the GlobalOverlayAccessIO at present. 
-    static final String REFERENCE_ENDPOINTS_WITH_BASEPATHS = "reference-endpoints-with-basepaths";
-    static final String BASE_STORE = "base-store";
-    static final String SECRET_KEY = "secret-key";
-    static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
-    static final String REMOTE_STORE_NAME = "remote-store-name";
-    static final String REMOTE_STORE_URL = "remote-store-url";
-    
-    protected StorageIO<DvObject> baseStore = null;
-    protected String path = null;
-    private String baseUrl = null;
-
-    protected static HttpClientContext localContext = HttpClientContext.create();
-    protected PoolingHttpClientConnectionManager cm = null;
-    CloseableHttpClient httpclient = null;
-    protected int timeout = 1200;
-    protected RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout * 1000)
-            .setConnectionRequestTimeout(timeout * 1000).setSocketTimeout(timeout * 1000)
-            .setCookieSpec(CookieSpecs.STANDARD).setExpectContinueEnabled(true).build();
-    protected static boolean trustCerts = false;
-    protected int httpConcurrency = 4;
+    String baseUrl = null;
 
     public RemoteOverlayAccessIO() {
+        super();
     }
     
     public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId) throws IOException {
         super(dvObject, req, driverId);
         this.setIsLocalFile(false);
+        configureRemoteEndpoints();
         configureStores(req, driverId, null);
         logger.fine("Parsing storageidentifier: " + dvObject.getStorageIdentifier());
         path = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
@@ -106,6 +64,7 @@ public RemoteOverlayAccessIO(T dvObject, DataAccessRequest req, String driverId)
     public RemoteOverlayAccessIO(String storageLocation, String driverId) throws IOException {
         super(null, null, driverId);
         this.setIsLocalFile(false);
+        configureRemoteEndpoints();
         configureStores(null, driverId, storageLocation);
 
         path = storageLocation.substring(storageLocation.lastIndexOf("//") + 2);
@@ -296,105 +255,12 @@ public void delete() throws IOException {
 
     }
 
-    @Override
-    public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException {
-        return baseStore.openAuxChannel(auxItemTag, options);
-    }
-
-    @Override
-    public boolean isAuxObjectCached(String auxItemTag) throws IOException {
-        return baseStore.isAuxObjectCached(auxItemTag);
-    }
-
-    @Override
-    public long getAuxObjectSize(String auxItemTag) throws IOException {
-        return baseStore.getAuxObjectSize(auxItemTag);
-    }
-
-    @Override
-    public Path getAuxObjectAsPath(String auxItemTag) throws IOException {
-        return baseStore.getAuxObjectAsPath(auxItemTag);
-    }
-
-    @Override
-    public void backupAsAux(String auxItemTag) throws IOException {
-        baseStore.backupAsAux(auxItemTag);
-    }
-
-    @Override
-    public void revertBackupAsAux(String auxItemTag) throws IOException {
-        baseStore.revertBackupAsAux(auxItemTag);
-    }
-
-    @Override
-    // this method copies a local filesystem Path into this DataAccess Auxiliary
-    // location:
-    public void savePathAsAux(Path fileSystemPath, String auxItemTag) throws IOException {
-        baseStore.savePathAsAux(fileSystemPath, auxItemTag);
-    }
-
-    @Override
-    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Long filesize) throws IOException {
-        baseStore.saveInputStreamAsAux(inputStream, auxItemTag, filesize);
-    }
-
-    /**
-     * @param inputStream InputStream we want to save
-     * @param auxItemTag  String representing this Auxiliary type ("extension")
-     * @throws IOException if anything goes wrong.
-     */
-    @Override
-    public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException {
-        baseStore.saveInputStreamAsAux(inputStream, auxItemTag);
-    }
-
-    @Override
-    public List<String> listAuxObjects() throws IOException {
-        return baseStore.listAuxObjects();
-    }
-
-    @Override
-    public void deleteAuxObject(String auxItemTag) throws IOException {
-        baseStore.deleteAuxObject(auxItemTag);
-    }
-
-    @Override
-    public void deleteAllAuxObjects() throws IOException {
-        baseStore.deleteAllAuxObjects();
-    }
-
-    @Override
-    public String getStorageLocation() throws IOException {
-        String fullStorageLocation = dvObject.getStorageIdentifier();
-        logger.fine("storageidentifier: " + fullStorageLocation);
-        int driverIndex = fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR);
-        if (driverIndex >= 0) {
-            fullStorageLocation = fullStorageLocation
-                    .substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
-        }
-        if (this.getDvObject() instanceof Dataset) {
-            throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
-        } else if (this.getDvObject() instanceof DataFile) {
-            fullStorageLocation = StorageIO.getDriverPrefix(this.driverId) + fullStorageLocation;
-        } else if (dvObject instanceof Dataverse) {
-            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
-        }
-        logger.fine("fullStorageLocation: " + fullStorageLocation);
-        return fullStorageLocation;
-    }
-
     @Override
     public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
         throw new UnsupportedDataAccessOperationException(
                 "RemoteOverlayAccessIO: this is a remote DataAccess IO object, it has no local filesystem path associated with it.");
     }
 
-    @Override
-    public boolean exists() {
-        logger.fine("Exists called");
-        return (retrieveSizeFromMedia() != -1);
-    }
-
     @Override
     public WritableByteChannel getWriteChannel() throws UnsupportedDataAccessOperationException {
         throw new UnsupportedDataAccessOperationException(
@@ -407,11 +273,6 @@ public OutputStream getOutputStream() throws UnsupportedDataAccessOperationExcep
                 "RemoteOverlayAccessIO: there are no output Streams associated with S3 objects.");
     }
 
-    @Override
-    public InputStream getAuxFileAsInputStream(String auxItemTag) throws IOException {
-        return baseStore.getAuxFileAsInputStream(auxItemTag);
-    }
-
     @Override
     public boolean downloadRedirectEnabled() {
         String optionValue = getConfigParam(StorageIO.DOWNLOAD_REDIRECT);
@@ -443,103 +304,12 @@ public String generateTemporaryDownloadUrl(String auxiliaryTag, String auxiliary
         }
     }
 
-    int getUrlExpirationMinutes() {
-        String optionValue = getConfigParam(URL_EXPIRATION_MINUTES);
-        if (optionValue != null) {
-            Integer num;
-            try {
-                num = Integer.parseInt(optionValue);
-            } catch (NumberFormatException ex) {
-                num = null;
-            }
-            if (num != null) {
-                return num;
-            }
-        }
-        return 60;
-    }
-
-    protected void configureStores(DataAccessRequest req, String driverId, String storageLocation) throws IOException {
-        configureEndpoints();
-        
-
-        if (baseStore == null) {
-            String baseDriverId = getBaseStoreIdFor(driverId);
-            String fullStorageLocation = null;
-            String baseDriverType = getConfigParamForDriver(baseDriverId, StorageIO.TYPE,
-                    DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
-
-            if (dvObject instanceof Dataset) {
-                baseStore = DataAccess.getStorageIO(dvObject, req, baseDriverId);
-            } else {
-                if (this.getDvObject() != null) {
-                    fullStorageLocation = getStoragePath();
-
-                    // S3 expects <id>://<bucketname>/<key>
-                    switch (baseDriverType) {
-                    case DataAccess.S3:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + getConfigParamForDriver(baseDriverId, S3AccessIO.BUCKET_NAME) + "/"
-                                + fullStorageLocation;
-                        break;
-                    case DataAccess.FILE:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + getConfigParamForDriver(baseDriverId, FileAccessIO.DIRECTORY, "/tmp/files")
-                                + "/" + fullStorageLocation;
-                        break;
-                    default:
-                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
-                                + getConfigParamForDriver(baseDriverId, StorageIO.TYPE));
-                        throw new IOException("Not supported");
-                    }
-
-                } else if (storageLocation != null) {
-                    // <remoteDriverId>://<baseStorageIdentifier>//<baseUrlPath>
-                    // remoteDriverId:// is removed if coming through directStorageIO
-                    int index = storageLocation.indexOf(DataAccess.SEPARATOR);
-                    if (index > 0) {
-                        storageLocation = storageLocation.substring(index + DataAccess.SEPARATOR.length());
-                    }
-                    // The base store needs the baseStoreIdentifier and not the relative URL (if it exists)
-                    int endOfId = storageLocation.indexOf("//");
-                    fullStorageLocation = (endOfId>-1) ? storageLocation.substring(0, endOfId) : storageLocation;
-
-                    switch (baseDriverType) {
-                    case DataAccess.S3:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + getConfigParamForDriver(baseDriverId, S3AccessIO.BUCKET_NAME) + "/"
-                                + fullStorageLocation;
-                        break;
-                    case DataAccess.FILE:
-                        fullStorageLocation = baseDriverId + DataAccess.SEPARATOR
-                                + getConfigParamForDriver(baseDriverId, FileAccessIO.DIRECTORY, "/tmp/files")
-                                + "/" + fullStorageLocation;
-                        break;
-                    default:
-                        logger.warning("Not Supported: " + this.getClass().getName() + " store with base store type: "
-                                + getConfigParamForDriver(baseDriverId, StorageIO.TYPE));
-                        throw new IOException("Not supported");
-                    }
-                }
-                baseStore = DataAccess.getDirectStorageIO(fullStorageLocation);
-            }
-            if (baseDriverType.contentEquals(DataAccess.S3)) {
-                ((S3AccessIO<?>) baseStore).setMainDriver(false);
-            }
-        }
-        remoteStoreName = getConfigParam(REMOTE_STORE_NAME);
-        try {
-            remoteStoreUrl = new URL(getConfigParam(REMOTE_STORE_URL));
-        } catch (MalformedURLException mfue) {
-            logger.fine("Unable to read remoteStoreUrl for driver: " + this.driverId);
-        }
-    }
 
     /** This endpoint configures all the endpoints the store is allowed to reference data from. At present, the RemoteOverlayAccessIO only supports a single endpoint but
      * the derived GlobusOverlayAccessIO can support multiple endpoints.
      * @throws IOException
      */
-    protected void configureEndpoints() throws IOException {
+    protected void configureRemoteEndpoints() throws IOException {
         baseUrl = getConfigParam(BASE_URL);
         if (baseUrl == null) {
             //Will accept the first endpoint using the newer setting
@@ -560,70 +330,6 @@ protected void configureEndpoints() throws IOException {
         }
     }
 
-    // Convenience method to assemble the path, starting with the DOI
-    // authority/identifier/, that is needed to create a base store via
-    // DataAccess.getDirectStorageIO - the caller has to add the store type specific
-    // prefix required.
-    protected String getStoragePath() throws IOException {
-        String fullStoragePath = dvObject.getStorageIdentifier();
-        logger.fine("storageidentifier: " + fullStoragePath);
-        int driverIndex = fullStoragePath.lastIndexOf(DataAccess.SEPARATOR);
-        if (driverIndex >= 0) {
-            fullStoragePath = fullStoragePath
-                    .substring(fullStoragePath.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
-        }
-        int suffixIndex = fullStoragePath.indexOf("//");
-        if (suffixIndex >= 0) {
-            fullStoragePath = fullStoragePath.substring(0, suffixIndex);
-        }
-        if (this.getDvObject() instanceof Dataset) {
-            fullStoragePath = this.getDataset().getAuthorityForFileStorage() + "/"
-                    + this.getDataset().getIdentifierForFileStorage() + "/" + fullStoragePath;
-        } else if (this.getDvObject() instanceof DataFile) {
-            fullStoragePath = this.getDataFile().getOwner().getAuthorityForFileStorage() + "/"
-                    + this.getDataFile().getOwner().getIdentifierForFileStorage() + "/" + fullStoragePath;
-        } else if (dvObject instanceof Dataverse) {
-            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
-        }
-        logger.fine("fullStoragePath: " + fullStoragePath);
-        return fullStoragePath;
-    }
-
-    public CloseableHttpClient getSharedHttpClient() {
-        if (httpclient == null) {
-            try {
-                initHttpPool();
-                httpclient = HttpClients.custom().setConnectionManager(cm).setDefaultRequestConfig(config).build();
-
-            } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException ex) {
-                logger.warning(ex.getMessage());
-            }
-        }
-        return httpclient;
-    }
-
-    private void initHttpPool() throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException {
-        if (trustCerts) {
-            // use the TrustSelfSignedStrategy to allow Self Signed Certificates
-            SSLContext sslContext;
-            SSLConnectionSocketFactory connectionFactory;
-
-            sslContext = SSLContextBuilder.create().loadTrustMaterial(new TrustAllStrategy()).build();
-            // create an SSL Socket Factory to use the SSLContext with the trust self signed
-            // certificate strategy
-            // and allow all hosts verifier.
-            connectionFactory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
-
-            Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
-                    .register("https", connectionFactory).build();
-            cm = new PoolingHttpClientConnectionManager(registry);
-        } else {
-            cm = new PoolingHttpClientConnectionManager();
-        }
-        cm.setDefaultMaxPerRoute(httpConcurrency);
-        cm.setMaxTotal(httpConcurrency > 20 ? httpConcurrency : 20);
-    }
-
     @Override
     public void savePath(Path fileSystemPath) throws IOException {
         throw new UnsupportedDataAccessOperationException(
@@ -660,13 +366,4 @@ static boolean isValidIdentifier(String driverId, String storageId) {
         }
         return true;
     }
-
-    public static String getBaseStoreIdFor(String driverId) {
-        return getConfigParamForDriver(driverId, BASE_STORE);
-    }
-
-    @Override
-    public List<String> cleanUp(Predicate<String> filter, boolean dryRun) throws IOException {
-        return baseStore.cleanUp(filter, dryRun);
-    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java
index 5affc01aff0..1c371881ba6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/RemoteOverlayAccessIOTest.java
@@ -8,7 +8,6 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.GlobalId;
-import edu.harvard.iq.dataverse.GlobalIdServiceBean;
 import edu.harvard.iq.dataverse.mocks.MocksFactory;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 

From 2500bccc5fa438bf2dff4e5aa887e816099a51e3 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 15 Nov 2023 14:04:22 -0500
Subject: [PATCH 737/922] assert current /bag-info.txt behavior #8760

Also, add a superuser-only API for downloading files
(such as bags) from the file system so we can make
assertions about them in our tests.
---
 .../iq/dataverse/api/AbstractApiBean.java     |   7 ++
 .../edu/harvard/iq/dataverse/api/Admin.java   |  25 ++++-
 .../edu/harvard/iq/dataverse/api/BagIT.java   | 101 +++++++++++++++++-
 3 files changed, 128 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
index 027f9e0fcb1..58565bcc9d6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
@@ -45,11 +45,13 @@
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.json.JsonParser;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import edu.harvard.iq.dataverse.validation.PasswordValidatorServiceBean;
+import java.io.InputStream;
 import java.net.URI;
 import java.util.Arrays;
 import java.util.Collections;
@@ -726,6 +728,11 @@ protected Response ok(String data, MediaType mediaType, String downloadFilename)
         return res.build();
     }
 
+    protected Response ok(InputStream inputStream) {
+        ResponseBuilder res = Response.ok().entity(inputStream).type(MediaType.valueOf(FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT));
+        return res.build();
+    }
+
     protected Response created( String uri, JsonObjectBuilder bld ) {
         return Response.created( URI.create(uri) )
                 .entity( Json.createObjectBuilder()
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index fd3b9a89e54..684ed32dff8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -107,6 +107,7 @@
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 
@@ -2425,5 +2426,27 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
         
         return ok(Json.createObjectBuilder().add(ExternalToolHandler.SIGNED_URL, signedUrl));
     }
- 
+
+    /**
+     * For testing only. Download a file from the file system.
+     */
+    @GET
+    @AuthRequired
+    @Path("/localfile")
+    public Response getLocalFile(@Context ContainerRequestContext crc, @QueryParam("pathToFile") String pathToFile) {
+        try {
+            AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+            if (!user.isSuperuser()) {
+                return error(Response.Status.FORBIDDEN, "Superusers only.");
+            }
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        try {
+            return ok(new FileInputStream(pathToFile));
+        } catch (IOException ex) {
+            return error(Status.BAD_REQUEST, ex.toString());
+        }
+    }
+
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java
index e7210bc45a9..fae9cf95156 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java
@@ -1,17 +1,32 @@
 package edu.harvard.iq.dataverse.api;
 
-import io.restassured.RestAssured;
-import io.restassured.response.Response;
 import edu.harvard.iq.dataverse.engine.command.impl.LocalSubmitToArchiveCommand;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import io.restassured.RestAssured;
+import static io.restassured.RestAssured.given;
+import io.restassured.response.Response;
 import static jakarta.ws.rs.core.Response.Status.CREATED;
 import static jakarta.ws.rs.core.Response.Status.OK;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Enumeration;
+import java.util.Scanner;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 import org.junit.jupiter.api.AfterAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
 public class BagIT {
 
+    static String bagitExportDir = "/tmp";
+
     @BeforeAll
     public static void setUpClass() {
 
@@ -25,14 +40,14 @@ public static void setUpClass() {
         setArchiverSettings.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
-        Response setBagItLocalPath = UtilIT.setSetting(":BagItLocalPath", "/tmp");
+        Response setBagItLocalPath = UtilIT.setSetting(":BagItLocalPath", bagitExportDir);
         setBagItLocalPath.then().assertThat()
                 .statusCode(OK.getStatusCode());
 
     }
 
     @Test
-    public void testBagItExport() {
+    public void testBagItExport() throws IOException {
 
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
@@ -63,6 +78,78 @@ public void testBagItExport() {
         archiveDataset.prettyPrint();
         archiveDataset.then().assertThat().statusCode(OK.getStatusCode());
 
+        // spaceName comes from LocalSubmitToArchiveCommand
+        String spaceName = datasetPid.replace(':', '-').replace('/', '-')
+                .replace('.', '-').toLowerCase();
+        // spacename: doi-10-5072-fk2-fosg5q
+
+        String pathToZip = bagitExportDir + "/" + spaceName + "v1.0" + ".zip";
+
+        try {
+            // give the bag time to generate
+            Thread.sleep(3000);
+        } catch (InterruptedException ex) {
+        }
+
+        // A bag could look like this:
+        //doi-10-5072-FK2-DKUTDUv-1-0/data/
+        //doi-10-5072-FK2-DKUTDUv-1-0/data/Darwin's Finches/
+        //doi-10-5072-FK2-DKUTDUv-1-0/metadata/
+        //doi-10-5072-FK2-DKUTDUv-1-0/metadata/pid-mapping.txt
+        //doi-10-5072-FK2-DKUTDUv-1-0/manifest-md5.txt
+        //doi-10-5072-FK2-DKUTDUv-1-0/bagit.txt
+        //doi-10-5072-FK2-DKUTDUv-1-0/metadata/oai-ore.jsonld
+        //doi-10-5072-FK2-DKUTDUv-1-0/metadata/datacite.xml
+        //doi-10-5072-FK2-DKUTDUv-1-0/bag-info.txt
+        // ---
+        // bag-info.txt could look like this:
+        //Contact-Name: Finch, Fiona
+        //Contact-Email: finch@mailinator.com
+        //Source-Organization: Dataverse Installation (<Site Url>)
+        //Organization-Address: <Full address>
+        //Organization-Email: <Email address>
+        //External-Description: Darwin's finches (also known as the Galápagos finches) are a group of about
+        // fifteen species of passerine birds.
+        //Bagging-Date: 2023-11-14
+        //External-Identifier: https://doi.org/10.5072/FK2/LZIGBC
+        //Bag-Size: 0 bytes
+        //Payload-Oxum: 0.0
+        //Internal-Sender-Identifier: Root:Darwin's Finches
+        Response downloadBag = downloadLocalFile(pathToZip, apiToken);
+        downloadBag.then().assertThat().statusCode(OK.getStatusCode());
+        Path outputPath = Paths.get("/tmp/foo.zip");
+        java.nio.file.Files.copy(downloadBag.getBody().asInputStream(), outputPath, StandardCopyOption.REPLACE_EXISTING);
+
+        ZipFile zipFile = new ZipFile(outputPath.toString());
+        Enumeration<? extends ZipEntry> entries = zipFile.entries();
+        String sourceOrg = null;
+        String orgAddress = null;
+        String orgEmail = null;
+        while (entries.hasMoreElements()) {
+            ZipEntry entry = entries.nextElement();
+            String name = entry.getName();
+            System.out.println("name: " + name);
+            if (name.endsWith("bag-info.txt")) {
+                InputStream stream = zipFile.getInputStream(entry);
+                Scanner s = new Scanner(stream).useDelimiter("\\A");
+                String result = s.hasNext() ? s.next() : "";
+                System.out.println("result: " + result);
+                String[] lines = result.split("\n");
+                for (String line : lines) {
+                    if (line.startsWith("Source-Organization")) {
+                        sourceOrg = line;
+                    } else if (line.startsWith("Organization-Address")) {
+                        orgAddress = line;
+                    } else if (line.startsWith("Organization-Email")) {
+                        orgEmail = line;
+                    } else {
+                    }
+                }
+            }
+        }
+        assertEquals("Source-Organization: Dataverse Installation (<Site Url>)", sourceOrg.trim());
+        assertEquals("Organization-Address: <Full address>", orgAddress.trim());
+        assertEquals("Organization-Email: <Email address>", orgEmail.trim());
     }
 
     @AfterAll
@@ -75,4 +162,10 @@ public static void tearDownClass() {
 
     }
 
+    static Response downloadLocalFile(String pathToFile, String apiToken) {
+        return given()
+                .header("X-Dataverse-key", apiToken)
+                .get("/api/admin/localfile?pathToFile=" + pathToFile);
+    }
+
 }

From 7240e870d35fda4ec96a4ee0e0b488a9c4fc3d4f Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 15 Nov 2023 16:03:15 -0500
Subject: [PATCH 738/922] configurable BagIt source org name, address, email
 #8760

These values were used while testing:

DATAVERSE_BAGIT_SOURCEORG_NAME=LibraScholar
DATAVERSE_BAGIT_SOURCEORG_ADDRESS=123 Wisdom Way\nCambridge, MA\nUSA
DATAVERSE_BAGIT_SOURCEORG_EMAIL=hello@dataverse.librascholar.edu
---
 .../iq/dataverse/settings/JvmSettings.java        |  7 +++++++
 .../iq/dataverse/util/bagit/BagGenerator.java     | 15 ++++++++++-----
 src/main/java/propertyFiles/Bundle.properties     |  4 ----
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index cc3272413c7..2f59350906c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -150,6 +150,13 @@ public enum JvmSettings {
     SCOPE_NETCDF(PREFIX, "netcdf"),
     GEO_EXTRACT_S3_DIRECT_UPLOAD(SCOPE_NETCDF, "geo-extract-s3-direct-upload"),
 
+    // BAGIT SETTINGS
+    SCOPE_BAGIT(PREFIX, "bagit"),
+    SCOPE_BAGIT_SOURCEORG(SCOPE_BAGIT, "sourceorg"),
+    BAGIT_SOURCE_ORG_NAME(SCOPE_BAGIT_SOURCEORG, "name"),
+    BAGIT_SOURCEORG_ADDRESS(SCOPE_BAGIT_SOURCEORG, "address"),
+    BAGIT_SOURCEORG_EMAIL(SCOPE_BAGIT_SOURCEORG, "email"),
+
     ;
 
     private static final String SCOPE_SEPARATOR = ".";
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java
index baba1a0cb43..b7c44014b80 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagGenerator.java
@@ -74,7 +74,9 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFile.ChecksumType;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.util.json.JsonLDTerm;
+import java.util.Optional;
 
 public class BagGenerator {
 
@@ -822,17 +824,20 @@ private String generateInfoFile() {
             logger.warning("No contact info available for BagIt Info file");
         }
 
-        info.append("Source-Organization: " + BundleUtil.getStringFromBundle("bagit.sourceOrganization"));
+        String orgName = JvmSettings.BAGIT_SOURCE_ORG_NAME.lookupOptional(String.class).orElse("Dataverse Installation (<Site Url>)");
+        String orgAddress = JvmSettings.BAGIT_SOURCEORG_ADDRESS.lookupOptional(String.class).orElse("<Full address>");
+        String orgEmail = JvmSettings.BAGIT_SOURCEORG_EMAIL.lookupOptional(String.class).orElse("<Email address>");
+
+        info.append("Source-Organization: " + orgName);
         // ToDo - make configurable
         info.append(CRLF);
 
-        info.append("Organization-Address: " + WordUtils.wrap(
-                BundleUtil.getStringFromBundle("bagit.sourceOrganizationAddress"), 78, CRLF + " ", true));
+        info.append("Organization-Address: " + WordUtils.wrap(orgAddress, 78, CRLF + " ", true));
+
         info.append(CRLF);
 
         // Not a BagIt standard name
-        info.append(
-                "Organization-Email: " + BundleUtil.getStringFromBundle("bagit.sourceOrganizationEmail"));
+        info.append("Organization-Email: " + orgEmail);
         info.append(CRLF);
 
         info.append("External-Description: ");
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 79887f7e76c..972e5e35601 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2379,10 +2379,6 @@ api.prov.error.freeformMissingJsonKey=The JSON object you send must have a key c
 api.prov.error.freeformNoText=No provenance free form text available for this file.
 api.prov.error.noDataFileFound=Could not find a file based on ID.
 
-bagit.sourceOrganization=Dataverse Installation (<Site Url>)
-bagit.sourceOrganizationAddress=<Full address>
-bagit.sourceOrganizationEmail=<Email address>
-
 bagit.checksum.validation.error=Invalid checksum for file "{0}". Manifest checksum={2}, calculated checksum={3}, type={1}
 bagit.checksum.validation.exception=Error while calculating checksum for file "{0}". Checksum type={1}, error={2}
 bagit.validation.bag.file.not.found=Invalid BagIt package: "{0}"

From b2c62510e71e6436c2905796b9cc6a24a04b35d0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 17 Nov 2023 14:06:06 -0500
Subject: [PATCH 739/922] add docs and release note for bag-info.txt config
 #8760

---
 doc/release-notes/8760-bagit.md               | 15 ++++++
 .../source/installation/config.rst            | 46 +++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 doc/release-notes/8760-bagit.md

diff --git a/doc/release-notes/8760-bagit.md b/doc/release-notes/8760-bagit.md
new file mode 100644
index 00000000000..30601857309
--- /dev/null
+++ b/doc/release-notes/8760-bagit.md
@@ -0,0 +1,15 @@
+For BagIT export, it is now possible to configure the following information in bag-info.txt...
+
+Source-Organization: Harvard Dataverse
+Organization-Address: 1737 Cambridge Street, Cambridge, MA, USA
+Organization-Email: support@dataverse.harvard.edu
+
+... using new JVM/MPCONFIG options:
+
+- dataverse.bagit.sourceorg.name
+- dataverse.bagit.sourceorg.address
+- dataverse.bagit.sourceorg.email
+
+Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.
+
+For details, see https://dataverse-guide--10122.org.readthedocs.build/en/10122/installation/config.html#bag-info-txt
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 13a7367de44..df311fcdaca 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1608,6 +1608,25 @@ The workflow id returned in this call (or available by doing a GET of /api/admin
 
 Once these steps are taken, new publication requests will automatically trigger submission of an archival copy to the specified archiver, Chronopolis' DuraCloud component in this example. For Chronopolis, as when using the API, it is currently the admin's responsibility to snap-shot the DuraCloud space and monitor the result. Failure of the workflow, (e.g. if DuraCloud is unavailable, the configuration is wrong, or the space for this dataset already exists due to a prior publication action or use of the API), will create a failure message but will not affect publication itself.
 
+.. _bag-info.txt:
+
+Configuring bag-info.txt
+++++++++++++++++++++++++
+
+Out of the box, placeholder values like below will be placed in bag-info.txt:
+
+.. code-block:: text
+
+  Source-Organization: Dataverse Installation (<Site Url>)
+  Organization-Address: <Full address>
+  Organization-Email: <Email address>
+
+To customize these values for your institution, use the following JVM options:
+
+- :ref:`dataverse.bagit.sourceorg.name`
+- :ref:`dataverse.bagit.sourceorg.address`
+- :ref:`dataverse.bagit.sourceorg.email`
+
 Going Live: Launching Your Production Deployment
 ------------------------------------------------
 
@@ -2506,6 +2525,33 @@ See also :ref:`guestbook-at-request-api` in the API Guide, and .
 
 Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_FILES_GUESTBOOK_AT_REQUEST``.
 
+.. _dataverse.bagit.sourceorg.name:
+
+dataverse.bagit.sourceorg.name
+++++++++++++++++++++++++++++++
+
+The name for your institution that you'd like to appear in bag-info.txt. See :ref:`bag-info.txt`.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_BAGIT_SOURCEORG_NAME``.
+
+.. _dataverse.bagit.sourceorg.address:
+
+dataverse.bagit.sourceorg.address
++++++++++++++++++++++++++++++++++
+
+The mailing address for your institution that you'd like to appear in bag-info.txt. See :ref:`bag-info.txt`. The example in https://datatracker.ietf.org/doc/html/rfc8493 uses commas as separators: ``1 Main St., Cupertino, California, 11111``.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_BAGIT_SOURCEORG_ADDRESS``.
+
+.. _dataverse.bagit.sourceorg.email:
+
+dataverse.bagit.sourceorg.email
++++++++++++++++++++++++++++++++
+
+The email for your institution that you'd like to appear in bag-info.txt. See :ref:`bag-info.txt`.
+
+Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_BAGIT_SOURCEORG_EMAIL``.
+
 .. _feature-flags:
 
 Feature Flags

From fa6f850b28e8dea1dd2dff542814e29fd7865153 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 17 Nov 2023 16:07:30 -0500
Subject: [PATCH 740/922] limit to downloading from /tmp, add docs #8760

---
 doc/release-notes/8760-download-tmp-file.md   |  3 +++
 doc/sphinx-guides/source/api/changelog.rst    |  7 +++++
 doc/sphinx-guides/source/api/native-api.rst   | 10 +++++++
 .../edu/harvard/iq/dataverse/api/Admin.java   | 13 +++++++---
 .../edu/harvard/iq/dataverse/api/AdminIT.java | 26 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/BagIT.java   | 10 ++-----
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  7 +++++
 7 files changed, 64 insertions(+), 12 deletions(-)
 create mode 100644 doc/release-notes/8760-download-tmp-file.md

diff --git a/doc/release-notes/8760-download-tmp-file.md b/doc/release-notes/8760-download-tmp-file.md
new file mode 100644
index 00000000000..7623a91ac9a
--- /dev/null
+++ b/doc/release-notes/8760-download-tmp-file.md
@@ -0,0 +1,3 @@
+A new API has been added for testing purposes that allows files to be downloaded from /tmp.
+
+See
diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index d6742252d27..7d6545999ca 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -5,6 +5,13 @@ API Changelog
     :local:
     :depth: 1
 
+6.1
+---
+
+New
+~~~
+- **/api/admin/downloadTmpFile**: See :ref:`download-file-from-tmp`.
+
 6.0
 -----
 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 1992390410c..5b1e7410a4f 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -5349,6 +5349,16 @@ A curl example using an ``ID``
 
 Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`.
 
+.. _download-file-from-tmp:
+
+Download File from /tmp
+~~~~~~~~~~~~~~~~~~~~~~~
+
+As a superuser::
+
+    GET /api/admin/downloadTmpFile?fullyQualifiedPathToFile=/tmp/foo.txt
+
+Note that this API is probably only useful for testing.
 
 MyData
 ------
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index 684ed32dff8..4da1962853a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -123,6 +123,7 @@
 import jakarta.ws.rs.QueryParam;
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.core.StreamingOutput;
+import java.nio.file.Paths;
 
 /**
  * Where the secure, setup API calls live.
@@ -2428,12 +2429,12 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
     }
 
     /**
-     * For testing only. Download a file from the file system.
+     * For testing only. Download a file from /tmp.
      */
     @GET
     @AuthRequired
-    @Path("/localfile")
-    public Response getLocalFile(@Context ContainerRequestContext crc, @QueryParam("pathToFile") String pathToFile) {
+    @Path("/downloadTmpFile")
+    public Response downloadTmpFile(@Context ContainerRequestContext crc, @QueryParam("fullyQualifiedPathToFile") String fullyQualifiedPathToFile) {
         try {
             AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
             if (!user.isSuperuser()) {
@@ -2442,8 +2443,12 @@ public Response getLocalFile(@Context ContainerRequestContext crc, @QueryParam("
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
+        java.nio.file.Path normalizedPath = Paths.get(fullyQualifiedPathToFile).normalize();
+        if (!normalizedPath.toString().startsWith("/tmp")) {
+            return error(Status.BAD_REQUEST, "Path must begin with '/tmp' but after normalization was '" + normalizedPath +"'.");
+        }
         try {
-            return ok(new FileInputStream(pathToFile));
+            return ok(new FileInputStream(fullyQualifiedPathToFile));
         } catch (IOException ex) {
             return error(Status.BAD_REQUEST, ex.toString());
         }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
index 0c5de662e8a..91ba67b10ff 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
@@ -854,6 +854,32 @@ public void testBannerMessages(){
         
     }
 
+    /**
+     * For a successful download from /tmp, see BagIT. Here we are doing error
+     * checking.
+     */
+    @Test
+    public void testDownloadTmpFile() throws IOException {
+
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response tryToDownloadAsNonSuperuser = UtilIT.downloadTmpFile("/tmp/foo", apiToken);
+        tryToDownloadAsNonSuperuser.then().assertThat().statusCode(FORBIDDEN.getStatusCode());
+
+        Response toggleSuperuser = UtilIT.makeSuperUser(username);
+        toggleSuperuser.then().assertThat()
+                .statusCode(OK.getStatusCode());
+
+        Response tryToDownloadEtcPasswd = UtilIT.downloadTmpFile("/etc/passwd", apiToken);
+        tryToDownloadEtcPasswd.then().assertThat()
+                .statusCode(BAD_REQUEST.getStatusCode())
+                .body("status", equalTo("ERROR"))
+                .body("message", equalTo("Path must begin with '/tmp' but after normalization was '/etc/passwd'."));
+    }
+
     private String createTestNonSuperuserApiToken() {
         Response createUserResponse = UtilIT.createRandomUser();
         createUserResponse.then().assertThat().statusCode(OK.getStatusCode());
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java
index fae9cf95156..28f7fa28328 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/BagIT.java
@@ -115,7 +115,7 @@ public void testBagItExport() throws IOException {
         //Bag-Size: 0 bytes
         //Payload-Oxum: 0.0
         //Internal-Sender-Identifier: Root:Darwin's Finches
-        Response downloadBag = downloadLocalFile(pathToZip, apiToken);
+        Response downloadBag = UtilIT.downloadTmpFile(pathToZip, apiToken);
         downloadBag.then().assertThat().statusCode(OK.getStatusCode());
         Path outputPath = Paths.get("/tmp/foo.zip");
         java.nio.file.Files.copy(downloadBag.getBody().asInputStream(), outputPath, StandardCopyOption.REPLACE_EXISTING);
@@ -162,10 +162,4 @@ public static void tearDownClass() {
 
     }
 
-    static Response downloadLocalFile(String pathToFile, String apiToken) {
-        return given()
-                .header("X-Dataverse-key", apiToken)
-                .get("/api/admin/localfile?pathToFile=" + pathToFile);
-    }
-
-}
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index e3a7fd0cfc3..6abfb10c4f6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3574,4 +3574,11 @@ static Response getDownloadSize(Integer datasetId,
         return requestSpecification
                 .get("/api/datasets/" + datasetId + "/versions/" + version + "/downloadsize");
     }
+
+    static Response downloadTmpFile(String fullyQualifiedPathToFile, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/admin/downloadTmpFile?fullyQualifiedPathToFile=" + fullyQualifiedPathToFile);
+    }
+
 }

From 06f6222ba785fa37890efa4156ec3e7988fe4ff5 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 20:29:47 -0500
Subject: [PATCH 741/922] more intermediate changes to the entity classes #8549

---
 .../edu/harvard/iq/dataverse/DvObject.java    | 28 +++++++++++++++++++
 .../iq/dataverse/DvObjectContainer.java       |  8 ++++--
 .../dataverse/ingest/IngestServiceBean.java   |  7 +++++
 3 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
index 9e7f3f3fe96..b86fabd0a07 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
@@ -2,6 +2,8 @@
 
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import edu.harvard.iq.dataverse.storageuse.StorageUse;
 
 import java.sql.Timestamp;
 import java.text.SimpleDateFormat;
@@ -156,6 +158,9 @@ public String visit(DataFile df) {
     
     private boolean identifierRegistered;
     
+    @Column(nullable = true)
+    private Long storageSize; 
+    
     private transient GlobalId globalId = null;
     
     @OneToMany(mappedBy = "dvObject", cascade = CascadeType.ALL, orphanRemoval = true)
@@ -177,6 +182,13 @@ public void setAlternativePersistentIndentifiers(Set<AlternativePersistentIdenti
      */
     private boolean previewImageAvailable;
     
+    @OneToOne(mappedBy = "definitionPoint",cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST}, orphanRemoval=true)
+    private StorageQuota storageQuota;
+    
+    @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE}, orphanRemoval=true)
+    private StorageUse storageUse;
+    
+    
     public boolean isPreviewImageAvailable() {
         return previewImageAvailable;
     }
@@ -458,6 +470,22 @@ public void setStorageIdentifier(String storageIdentifier) {
         this.storageIdentifier = storageIdentifier;
     }
     
+    public Long getStorageSize() {
+        return storageSize; 
+    }
+    
+    public void setStorageSize(Long storageSize) {
+        this.storageSize = storageSize; 
+    }
+    
+    public StorageQuota getStorageQuota() {
+        return storageQuota;
+    }
+    
+    public void setStorageQuota(StorageQuota storageQuota) {
+        this.storageQuota = storageQuota;
+    }
+
     /**
      * 
      * @param other 
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index f7d361d76f5..2f391e394fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -2,11 +2,15 @@
 
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import edu.harvard.iq.dataverse.storageuse.StorageUse;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import jakarta.persistence.CascadeType;
 import java.util.Locale;
 import java.util.Optional;
 
 import jakarta.persistence.MappedSuperclass;
+import jakarta.persistence.OneToOne;
 import org.apache.commons.lang3.StringUtils;
 
 /**
@@ -40,7 +44,7 @@ public boolean isEffectivelyPermissionRoot() {
     private String metadataLanguage=null;
     
     private Boolean guestbookAtRequest = null;
-    
+   
     public String getEffectiveStorageDriverId() {
         String id = storageDriver;
         if (StringUtils.isBlank(id)) {
@@ -160,5 +164,5 @@ public String getCurationLabelSetName() {
     public void setCurationLabelSetName(String setName) {
         this.externalLabelSetName = setName;
     }
-
+    
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index d6496a97b97..391f28674e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -174,6 +174,13 @@ public class IngestServiceBean {
     // It must be called before we attempt to permanently save the files in 
     // the database by calling the Save command on the dataset and/or version.
     
+    // There is way too much going on in this method. :(
+    public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
+            List<DataFile> newFiles,
+            DataFile fileToReplace,
+            boolean tabIngest) {
+        return saveAndAddFilesToDataset(version, newFiles, fileToReplace, tabIngest, null);
+    }
     public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
             List<DataFile> newFiles,
             DataFile fileToReplace,

From 8766932b6c086b1775e3faf8e19f411d83f87c07 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 21:09:12 -0500
Subject: [PATCH 742/922] extra logging

---
 .../iq/dataverse/search/SearchIncludeFragment.java       | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 400f10cc375..c579eb14b7e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -343,9 +343,10 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         setSolrErrorEncountered(false);
         
         try {
-            logger.fine("ATTENTION! query from user:   " + query);
-            logger.fine("ATTENTION! queryToPassToSolr: " + queryToPassToSolr);
-            logger.fine("ATTENTION! sort by: " + sortField);
+            logger.info("ATTENTION! query from user:   " + query);
+            logger.info("ATTENTION! queryToPassToSolr: " + queryToPassToSolr);
+            logger.info("ATTENTION! filterQueriesFinal: " + filterQueriesFinal);
+            logger.info("ATTENTION! sort by: " + sortField);
 
             /**
              * @todo Number of search results per page should be configurable -
@@ -408,6 +409,8 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                     }
                 }
                 filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
+                logger.info("second pass query: " + queryToPassToSolr);
+                logger.info("second pass filter query: "+filterQueriesFinalSecondPass.toString());
 
                 solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, null, sortOrder.toString(), 0, onlyDataRelatedToMe, 1, false, null, null, false, false);
 

From 552e7350cd7f9d9eb577b056e8d3eb414e8dc3cc Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 21:09:40 -0500
Subject: [PATCH 743/922] get quota command #8549

---
 .../impl/GetCollectionQuotaCommand.java       | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java
new file mode 100644
index 00000000000..f07fde9508e
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java
@@ -0,0 +1,45 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.util.BundleUtil;
+import java.io.IOException;
+import java.util.List;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author landreev
+ * The command doesn't do much. It's sole purpose is to check the permissions
+ * when it's called by the /api/dataverses/.../storage/quota api. 
+ */
+@RequiredPermissions(Permission.ManageDataversePermissions)
+public class GetCollectionQuotaCommand  extends AbstractCommand<Long> {
+
+    private static final Logger logger = Logger.getLogger(GetCollectionQuotaCommand.class.getCanonicalName());
+    
+    private final Dataverse dataverse;
+    
+    public GetCollectionQuotaCommand(DataverseRequest aRequest, Dataverse target) {
+        super(aRequest, target);
+        dataverse = target;
+    } 
+        
+    @Override
+    public Long execute(CommandContext ctxt) throws CommandException {
+               
+        if (dataverse != null && dataverse.getStorageQuota() != null) {
+            return dataverse.getStorageQuota().getAllocation();
+        }
+        
+        return null;
+    }    
+}
+
+    

From e4aea93f0ada3212d1116b13cd0b2ae8105100e1 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 21:20:29 -0500
Subject: [PATCH 744/922] extra logging

---
 .../edu/harvard/iq/dataverse/search/SearchIncludeFragment.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index c579eb14b7e..e5b5763efe6 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -345,7 +345,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         try {
             logger.info("ATTENTION! query from user:   " + query);
             logger.info("ATTENTION! queryToPassToSolr: " + queryToPassToSolr);
-            logger.info("ATTENTION! filterQueriesFinal: " + filterQueriesFinal);
+            logger.info("ATTENTION! filterQueriesFinal: " + filterQueriesFinal.toString());
             logger.info("ATTENTION! sort by: " + sortField);
 
             /**

From 2b8777990d008b31e61c4338f5b5e964e1f4a20d Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 21:21:17 -0500
Subject: [PATCH 745/922] new classes and instances #8549

---
 .../iq/dataverse/storageuse/StorageQuota.java | 118 ++++++++++++++++++
 .../iq/dataverse/storageuse/StorageUse.java   |  94 ++++++++++++++
 .../storageuse/StorageUseServiceBean.java     |  65 ++++++++++
 3 files changed, 277 insertions(+)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
new file mode 100644
index 00000000000..68ff6d95d00
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
@@ -0,0 +1,118 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.storageuse;
+
+import edu.harvard.iq.dataverse.DvObject;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.OneToOne;
+import java.io.Serializable;
+import java.util.logging.Logger;
+
+//import jakarta.persistence.*;
+
+/**
+ *
+ * @author landreev
+ * 
+ */
+@Entity
+public class StorageQuota implements Serializable {
+    private static final Logger logger = Logger.getLogger(StorageQuota.class.getCanonicalName()); 
+    
+    /**
+     * Only Collection quotas are supported, for now
+     */
+    
+    private static final long serialVersionUID = 1L;
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+    
+    /**
+     * For defining quotas for Users and/or Groups 
+     * (Not supported as of yet)
+     
+    @Column(nullable = true)
+    private String assigneeIdentifier;
+    */
+    
+    /**
+     * Could be changed to ManyToOne - if we wanted to be able to define separate
+     * quotas on the same collection for different users. (?)
+     * Whether we actually want to support the above is TBD. (possibly not)
+     * Only collection-wide quotas are supported for now. 
+     */
+    @OneToOne
+    @JoinColumn(name="definitionPoint_id", nullable=true)
+    private DvObject definitionPoint;
+    
+    @Column(nullable = true)
+    private Long allocation; 
+    
+    public StorageQuota() {}
+    
+    /*public String getAssigneeIdentifier() {
+        return assigneeIdentifier;
+    }
+
+    public void setAssigneeIdentifier(String assigneeIdentifier) {
+        this.assigneeIdentifier = assigneeIdentifier;
+    }*/
+    
+    public DvObject getDefinitionPoint() {
+        return definitionPoint;
+    }
+
+    public void setDefinitionPoint(DvObject definitionPoint) {
+        this.definitionPoint = definitionPoint;
+    }
+    
+    public Long getAllocation() {
+        return allocation; 
+    }
+    
+    public void setAllocation(Long allocation) {
+        this.allocation = allocation; 
+    }
+    
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += (id != null ? id.hashCode() : 0);
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        // TODO: Warning - this method won't work in the case the id fields are not set
+        if (!(object instanceof StorageQuota)) {
+            return false;
+        }
+        StorageQuota other = (StorageQuota) object;
+        if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return "edu.harvard.iq.dataverse.storageuse.StorageQuota[ id=" + id + " ]";
+    }
+    
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
new file mode 100644
index 00000000000..2633e3e026b
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
@@ -0,0 +1,94 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.storageuse;
+
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.DvObjectContainer;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.Id;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.NamedQueries;
+import jakarta.persistence.NamedQuery;
+import jakarta.persistence.OneToOne;
+import java.io.Serializable;
+
+/**
+ *
+ * @author landreev
+ */
+@NamedQueries({
+    @NamedQuery(name = "StorageUse.findByteSizeByDvContainerId",query = "SELECT su.sizeInBytes FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId "),
+    @NamedQuery(name = "StorageUse.findByDvContainerId",query = "SELECT su FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId ")
+})
+@Entity
+public class StorageUse implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+    @Id
+    @GeneratedValue(strategy = GenerationType.AUTO)
+    private Long id;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    @OneToOne
+    @JoinColumn(nullable=false)
+    private DvObject dvObjectContainer; 
+    
+    @Column
+    private Long sizeInBytes = null; 
+    
+    public StorageUse(DvObjectContainer dvObjectContainer, Long sizeInBytes) {
+        this.dvObjectContainer = dvObjectContainer;
+        this.sizeInBytes = sizeInBytes;
+    }
+    
+    public Long getSizeInBytes() {
+        return sizeInBytes; 
+    }
+    
+    public void setSizeInBytes(Long sizeInBytes) {
+        this.sizeInBytes = sizeInBytes;
+    }
+    
+    public void incrementSizeInBytes(Long sizeInBytes) {
+        this.sizeInBytes += sizeInBytes; 
+    }
+    
+    
+    @Override
+    public int hashCode() {
+        int hash = 0;
+        hash += (id != null ? id.hashCode() : 0);
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        // TODO: Warning - this method won't work in the case the id fields are not set
+        if (!(object instanceof StorageUse)) {
+            return false;
+        }
+        StorageUse other = (StorageUse) object;
+        if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) {
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return "edu.harvard.iq.dataverse.storageuse.StorageUse[ id=" + id + " ]";
+    }
+    
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
new file mode 100644
index 00000000000..fd04344c234
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
@@ -0,0 +1,65 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.storageuse;
+
+import edu.harvard.iq.dataverse.DataverseServiceBean;
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.DvObjectContainer;
+import jakarta.ejb.EJB;
+import jakarta.ejb.Stateless;
+import jakarta.inject.Named;
+import jakarta.persistence.EntityManager;
+import jakarta.persistence.PersistenceContext;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author landreev
+ */
+@Stateless
+@Named
+public class StorageUseServiceBean  implements java.io.Serializable {
+    private static final Logger logger = Logger.getLogger(StorageUseServiceBean.class.getCanonicalName());
+    @EJB
+    DataverseServiceBean dataverseService;
+    
+    @PersistenceContext(unitName = "VDCNet-ejbPU")
+    private EntityManager em;
+    
+    public StorageUse findByDvContainerId(Long dvObjectId) {
+        return em.createNamedQuery("StorageUse.findByDvContainerId", StorageUse.class).setParameter("dvObjectId", dvObjectId).getSingleResult();
+    }
+    
+    public Long findStorageSizeByDvContainerId(Long dvObjectId) {
+        return em.createNamedQuery("StorageUse.findByteSizeByDvContainerId", Long.class).setParameter("dvObjectId", dvObjectId).getSingleResult();
+    }
+    
+    public void incrementStorageSizeHierarchy(DvObjectContainer dvObject, Long filesize) {
+        incrementStorageSize(dvObject, filesize); 
+        DvObjectContainer parent = dvObject.getOwner();
+        while (parent != null) {
+            incrementStorageSize(parent, filesize);
+            parent = parent.getOwner();
+        }
+    }
+    
+    /**
+     * Should this be done in a new transaction?
+     * @param dvObject
+     * @param filesize 
+     */
+    public void incrementStorageSize(DvObjectContainer dvObject, Long filesize) {
+        StorageUse dvContainerSU = findByDvContainerId(dvObject.getId());
+        if (dvContainerSU != null) {
+            // @todo: named query
+            dvContainerSU.incrementSizeInBytes(filesize);
+            em.merge(dvContainerSU);
+        } else {
+            dvContainerSU = new StorageUse(dvObject, filesize); 
+            em.persist(dvContainerSU);
+        }
+    }
+    
+}

From 235b1b018a50fd099c983516b046c6847be41e48 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 21:44:47 -0500
Subject: [PATCH 746/922] A fix for the missing subtree filter query in the 2nd
 pass search query. #9635

---
 .../search/SearchIncludeFragment.java         | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index e5b5763efe6..1acd4b0f8a1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -282,7 +282,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         SolrQueryResponse solrQueryResponse = null;
         SolrQueryResponse solrQueryResponseSecondPass = null;
 
-        List<String> filterQueriesFinal = new ArrayList<>();
+        List<String> filterQueriesExtended = new ArrayList<>();
         
         if (dataverseAlias != null) {
             this.dataverse = dataverseService.findByAlias(dataverseAlias);
@@ -296,7 +296,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                  * @todo centralize this into SearchServiceBean
                  */
                 if (!isfilterQueryAlreadyInMap(filterDownToSubtree)){
-                    filterQueriesFinal.add(filterDownToSubtree);
+                    filterQueriesExtended.add(filterDownToSubtree);
                 }
 //                this.dataverseSubtreeContext = dataversePath;
             } else {
@@ -309,7 +309,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
             this.setRootDv(true);
         }
 
-        filterQueriesFinal.addAll(filterQueries);
+        filterQueriesExtended.addAll(filterQueries);
 
         /**
          * Add type queries, for the types (Dataverses, Datasets, Datafiles) 
@@ -323,7 +323,9 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         selectedTypesHumanReadable = combine(arr, " OR ");
         if (!selectedTypesHumanReadable.isEmpty()) {
             typeFilterQuery = SearchFields.TYPE + ":(" + selectedTypesHumanReadable + ")";
-        }        
+        } 
+        List<String> filterQueriesFinal = new ArrayList<>();
+        filterQueriesFinal.addAll(filterQueriesExtended);
         filterQueriesFinal.add(typeFilterQuery);
 
         if (page <= 1) {
@@ -343,10 +345,10 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         setSolrErrorEncountered(false);
         
         try {
-            logger.info("ATTENTION! query from user:   " + query);
-            logger.info("ATTENTION! queryToPassToSolr: " + queryToPassToSolr);
-            logger.info("ATTENTION! filterQueriesFinal: " + filterQueriesFinal.toString());
-            logger.info("ATTENTION! sort by: " + sortField);
+            logger.fine"ATTENTION! query from user:   " + query);
+            logger.fine("ATTENTION! queryToPassToSolr: " + queryToPassToSolr);
+            logger.fine("ATTENTION! filterQueriesFinal: " + filterQueriesFinal.toString());
+            logger.fine("ATTENTION! sort by: " + sortField);
 
             /**
              * @todo Number of search results per page should be configurable -
@@ -399,7 +401,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 // run a second search to obtain the numbers of the unselected types:
                 
                 List<String> filterQueriesFinalSecondPass = new ArrayList<>();
-                filterQueriesFinalSecondPass.addAll(filterQueries);
+                filterQueriesFinalSecondPass.addAll(filterQueriesExtended);
                    
                 arr = new String[3 - selectedTypesList.size()];
                 int c = 0; 
@@ -409,8 +411,8 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                     }
                 }
                 filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
-                logger.info("second pass query: " + queryToPassToSolr);
-                logger.info("second pass filter query: "+filterQueriesFinalSecondPass.toString());
+                logger.fine("second pass query: " + queryToPassToSolr);
+                logger.fine("second pass filter query: "+filterQueriesFinalSecondPass.toString());
 
                 solrQueryResponseSecondPass = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinalSecondPass, null, sortOrder.toString(), 0, onlyDataRelatedToMe, 1, false, null, null, false, false);
 

From ceeeaecb9d222c2d2073713cdd839dac2ab4a304 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 19 Nov 2023 21:47:30 -0500
Subject: [PATCH 747/922] typo. #9635

---
 .../edu/harvard/iq/dataverse/search/SearchIncludeFragment.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 1acd4b0f8a1..dd9cd78982a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -345,7 +345,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
         setSolrErrorEncountered(false);
         
         try {
-            logger.fine"ATTENTION! query from user:   " + query);
+            logger.fine("ATTENTION! query from user:   " + query);
             logger.fine("ATTENTION! queryToPassToSolr: " + queryToPassToSolr);
             logger.fine("ATTENTION! filterQueriesFinal: " + filterQueriesFinal.toString());
             logger.fine("ATTENTION! sort by: " + sortField);

From e4ede35ea8a57afc8830dc63619bed3b660da8ff Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 20 Nov 2023 09:37:27 -0500
Subject: [PATCH 748/922] #9464 fix logger reference

---
 .../engine/command/impl/ValidateDatasetJsonCommand.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
index ae1a89c3661..619740ddd89 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ValidateDatasetJsonCommand.java
@@ -21,7 +21,7 @@
 @RequiredPermissions(Permission.AddDataset)
 public class ValidateDatasetJsonCommand extends AbstractCommand<String> {
     
-    private static final Logger logger = Logger.getLogger(GetDatasetSchemaCommand.class.getCanonicalName());
+    private static final Logger logger = Logger.getLogger(ValidateDatasetJsonCommand.class.getCanonicalName());
     
     private final Dataverse dataverse;
     private final String datasetJson;

From d30ecfda14bd4adcafced8486d58507aba12c55f Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 25 Oct 2023 10:56:14 -0400
Subject: [PATCH 749/922] add S3 tests, LocalStack, MinIO #6783

Developers can now test S3 locally by using the Dockerized
development environment, which now includes both LocalStack
and MinIO. See S3AccessIT which executes API (end to end) tests.

In addition, a new integration test test class
(not an API test, the new kind launched with `mvn verify`)
has been added at S3AccessIOLocalstackIT. It uses Testcontainers
to spin up Localstack for S3 testing and does not require
Dataverse to be running.

Note that the format of docker-compose-dev.yml had to change to
allow for JVM options to be added.

Finally, docs were improved for listing and setting stores via API.
---
 conf/localstack/buckets.sh                    |   3 +
 doc/release-notes/6783-s3-tests.md            |   3 +
 .../source/admin/dataverses-datasets.rst      |   4 +
 docker-compose-dev.yml                        |  78 +++++-
 pom.xml                                       |   5 +
 .../harvard/iq/dataverse/api/S3AccessIT.java  | 228 +++++++++++++++---
 .../dataaccess/S3AccessIOLocalstackIT.java    | 153 ++++++++++++
 7 files changed, 436 insertions(+), 38 deletions(-)
 create mode 100755 conf/localstack/buckets.sh
 create mode 100644 doc/release-notes/6783-s3-tests.md
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIOLocalstackIT.java

diff --git a/conf/localstack/buckets.sh b/conf/localstack/buckets.sh
new file mode 100755
index 00000000000..fe940d9890d
--- /dev/null
+++ b/conf/localstack/buckets.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# https://stackoverflow.com/questions/53619901/auto-create-s3-buckets-on-localstack
+awslocal s3 mb s3://mybucket
diff --git a/doc/release-notes/6783-s3-tests.md b/doc/release-notes/6783-s3-tests.md
new file mode 100644
index 00000000000..1febb87aaed
--- /dev/null
+++ b/doc/release-notes/6783-s3-tests.md
@@ -0,0 +1,3 @@
+Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. See S3AccessIT which executes API (end to end) tests.
+
+In addition, a new integration test test class (not an API test, the new kind launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.
diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst
index 170807d3d67..37494c57fa1 100644
--- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst
+++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst
@@ -53,11 +53,15 @@ Configure a Dataverse Collection to Store All New Files in a Specific File Store
 To direct new files (uploaded when datasets are created or edited) for all datasets in a given Dataverse collection, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse collection on the Dataverse collection page. Only accessible to superusers. ::
  
     curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
+
+(Note that for ``dataverse.files.store1.label=MyLabel``, you should pass ``MyLabel``.)
     
 The current driver can be seen using::
 
     curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
 
+(Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.)
+
 and can be reset to the default store with::
 
     curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index bb0a4c95b12..769c24fb3a5 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -9,16 +9,38 @@ services:
     restart: on-failure
     user: payara
     environment:
-      - DATAVERSE_DB_HOST=postgres
-      - DATAVERSE_DB_PASSWORD=secret
-      - DATAVERSE_DB_USER=${DATAVERSE_DB_USER}
-      - ENABLE_JDWP=1
-      - DATAVERSE_FEATURE_API_BEARER_AUTH=1
-      - DATAVERSE_AUTH_OIDC_ENABLED=1
-      - DATAVERSE_AUTH_OIDC_CLIENT_ID=test
-      - DATAVERSE_AUTH_OIDC_CLIENT_SECRET=94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8
-      - DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL=http://keycloak.mydomain.com:8090/realms/test
-      - DATAVERSE_JSF_REFRESH_PERIOD=1
+      DATAVERSE_DB_HOST: postgres
+      DATAVERSE_DB_PASSWORD: secret
+      DATAVERSE_DB_USER: ${DATAVERSE_DB_USER}
+      ENABLE_JDWP: "1"
+      DATAVERSE_FEATURE_API_BEARER_AUTH: "1"
+      DATAVERSE_AUTH_OIDC_ENABLED: "1"
+      DATAVERSE_AUTH_OIDC_CLIENT_ID: test
+      DATAVERSE_AUTH_OIDC_CLIENT_SECRET: 94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8
+      DATAVERSE_AUTH_OIDC_AUTH_SERVER_URL: http://keycloak.mydomain.com:8090/realms/test
+      DATAVERSE_JSF_REFRESH_PERIOD: "1"
+      JVM_ARGS: -Ddataverse.files.storage-driver-id=file1
+        -Ddataverse.files.file1.type=file
+        -Ddataverse.files.file1.label=Filesystem
+        -Ddataverse.files.file1.directory=${STORAGE_DIR}/store
+        -Ddataverse.files.localstack1.type=s3
+        -Ddataverse.files.localstack1.label=LocalStack
+        -Ddataverse.files.localstack1.custom-endpoint-url=http://localstack:4566
+        -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2
+        -Ddataverse.files.localstack1.bucket-name=mybucket
+        -Ddataverse.files.localstack1.path-style-access=true
+        -Ddataverse.files.localstack1.upload-redirect=false
+        -Ddataverse.files.localstack1.access-key=default
+        -Ddataverse.files.localstack1.secret-key=default
+        -Ddataverse.files.minio1.type=s3
+        -Ddataverse.files.minio1.label=MinIO
+        -Ddataverse.files.minio1.custom-endpoint-url=http://minio:9000
+        -Ddataverse.files.minio1.custom-endpoint-region=us-east-1
+        -Ddataverse.files.minio1.bucket-name=mybucket
+        -Ddataverse.files.minio1.path-style-access=true
+        -Ddataverse.files.minio1.upload-redirect=false
+        -Ddataverse.files.minio1.access-key=minioadmin
+        -Ddataverse.files.minio1.secret-key=minioadmin
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4848:4848" # HTTP (Payara Admin Console)
@@ -156,6 +178,42 @@ services:
     networks:
       - dataverse
 
+  dev_localstack:
+    container_name: "dev_localstack"
+    hostname: "localstack"
+    image: localstack/localstack:2.3.2
+    restart: on-failure
+    ports:
+      - "127.0.0.1:4566:4566"
+    environment:
+      - DEBUG=${DEBUG-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOSTNAME_EXTERNAL=localstack
+    networks:
+      - dataverse
+    volumes:
+      - ./conf/localstack:/etc/localstack/init/ready.d
+    tmpfs:
+      - /localstack:mode=770,size=128M,uid=1000,gid=1000
+
+  dev_minio:
+    container_name: "dev_minio"
+    hostname: "minio"
+    image: minio/minio
+    restart: on-failure
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    networks:
+      - dataverse
+    volumes:
+      - minio_storage:/data
+    environment:
+      # these are the defaults but are here for clarity
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    command: server /data
+
 networks:
   dataverse:
     driver: bridge
diff --git a/pom.xml b/pom.xml
index 4d10073334f..34b0ad2e835 100644
--- a/pom.xml
+++ b/pom.xml
@@ -612,6 +612,11 @@
             <version>3.0.0</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
+            <scope>test</scope>
+        </dependency>
         <!--
             Brute force solution until we are on Jakarta EE 10.
             Otherwise, we get very cryptic errors about missing bundle files on test runs.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index 8b1e96f3622..891a54e87c6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -1,66 +1,238 @@
 package edu.harvard.iq.dataverse.api;
 
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.HeadBucketRequest;
+import com.amazonaws.services.s3.model.HeadBucketResult;
 import io.restassured.RestAssured;
+import static io.restassured.RestAssured.given;
 import io.restassured.path.json.JsonPath;
 import io.restassured.response.Response;
+import java.util.List;
 import java.util.logging.Logger;
+import static org.hamcrest.CoreMatchers.equalTo;
 
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.Matchers.startsWith;
+import org.junit.jupiter.api.Assertions;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 /**
- *  NOTE: This test WILL NOT pass if your installation is not configured for Amazon S3 storage.
- *  For S3 storage, you must set two jvm options: storage-driver-id and s3-bucket-name
- *  Refer to the guides or to https://github.com/IQSS/dataverse/issues/3921#issuecomment-319973245
- * @author bsilverstein
+ * This test requires services spun up in Docker.
  */
 public class S3AccessIT {
-    
+
     private static final Logger logger = Logger.getLogger(S3AccessIT.class.getCanonicalName());
 
+    public enum TypesOfS3 {
+        MINIO,
+        LOCALSTACK
+    };
+
+    static final String accessKey = "minioadmin";
+    static final String secretKey = "minioadmin";
+    static final String bucketName = "mybucket";
+    static String driverId;
+    static String driverLabel;
+    static AmazonS3 s3 = null;
+
     @BeforeAll
     public static void setUp() {
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
-        
+
+        TypesOfS3 typeToTest = TypesOfS3.LOCALSTACK;
+        typeToTest = TypesOfS3.MINIO;
+
+        switch (typeToTest) {
+            case LOCALSTACK -> {
+                driverId = "localstack1";
+                driverLabel = "LocalStack";
+                s3 = AmazonS3ClientBuilder.standard()
+                        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
+                        .withEndpointConfiguration(new EndpointConfiguration("s3.localhost.localstack.cloud:4566", Regions.US_EAST_2.getName())).build();
+            }
+            case MINIO -> {
+                driverId = "minio1";
+                driverLabel = "MinIO";
+                s3 = AmazonS3ClientBuilder.standard()
+                        // https://stackoverflow.com/questions/72205086/amazonss3client-throws-unknownhostexception-if-attempting-to-connect-to-a-local
+                        .withPathStyleAccessEnabled(Boolean.TRUE)
+                        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
+                        .withEndpointConfiguration(new EndpointConfiguration("http://localhost:9000", Regions.US_EAST_1.getName())).build();
+//                String location = s3.getBucketLocation(bucketName);
+//                if (location != "US") {
+//                    Bucket bucket = s3.createBucket(bucketName);
+//                }
+            }
+        }
+        System.out.println("buckets before attempting to create " + bucketName);
+        for (Bucket bucket : s3.listBuckets()) {
+            System.out.println("bucket: " + bucket);
+        }
+
+        // create bucket if it doesn't exist
+        // Note that we create the localstack bucket with conf/localstack/buckets.sh
+        // because we haven't figured out how to create it properly in Java.
+        // Perhaps it is missing ACLs.
+        try {
+            s3.headBucket(new HeadBucketRequest(bucketName));
+        } catch (AmazonS3Exception ex) {
+            s3.createBucket(bucketName);
+        }
+
+//        String location = s3.getBucketLocation(bucketName);
+////        HeadBucketRequest headBucketRequest;
+//        s3.headBucket(headBucketRequest);
+//        if (location != null && !"US".equals(location)) {
+//            System.out.println("Creating bucket. Location was " + location);
+//            Bucket createdBucket = s3.createBucket(bucketName);
+//            System.out.println("created bucket: " + createdBucket);
+//        }
+//        System.out.println("buckets after creating " + bucketName);
+//        for (Bucket bucket : s3.listBuckets()) {
+//            System.out.println("bucket: " + bucket);
+//        }
     }
-    
+
     @Test
     public void testAddDataFileS3Prefix() {
+        Response createSuperuser = UtilIT.createRandomUser();
+        String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
+        String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
+        UtilIT.makeSuperUser(superusername);
+        Response storageDrivers = listStorageDrivers(superuserApiToken);
+        storageDrivers.prettyPrint();
+        // TODO where is "Local/local" coming from?
+        String drivers = """
+{
+    "status": "OK",
+    "data": {
+        "LocalStack": "localstack1",
+        "MinIO": "minio1",
+        "Local": "local",
+        "Filesystem": "file1"
+    }
+}""";
+
         //create user who will make a dataverse/dataset
         Response createUser = UtilIT.createRandomUser();
         String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
-        
+
         Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
-        
+
+        Response originalStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        originalStorageDriver.prettyPrint();
+        originalStorageDriver.then().assertThat()
+                .body("data.message", equalTo("undefined"))
+                .statusCode(200);
+
+        Response setStorageDriverToS3 = setStorageDriver(dataverseAlias, driverLabel, superuserApiToken);
+        setStorageDriverToS3.prettyPrint();
+        setStorageDriverToS3.then().assertThat()
+                .statusCode(200);
+
+        Response updatedStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        updatedStorageDriver.prettyPrint();
+        updatedStorageDriver.then().assertThat()
+                .statusCode(200);
+
         Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
-        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
         createDatasetResponse.prettyPrint();
-        
-        //upload a tabular file via native, check storage id prefix for s3
+        createDatasetResponse.then().assertThat().statusCode(201);
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        String datasetPid = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
+        String datasetStorageIdentifier = datasetPid.substring(4);
+
+        Response getDatasetMetadata = UtilIT.nativeGet(datasetId, apiToken);
+        getDatasetMetadata.prettyPrint();
+        getDatasetMetadata.then().assertThat().statusCode(200);
+
+        //upload a tabular file via native, check storage id prefix for driverId
         String pathToFile = "scripts/search/data/tabular/1char";
         Response addFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
         addFileResponse.prettyPrint();
         addFileResponse.then().assertThat()
-                .body("data.files[0].dataFile.storageIdentifier", startsWith("s3://"));
-        
-        //clean up test dvobjects and user
-        Response deleteDataset = UtilIT.deleteDatasetViaNativeApi(datasetId, apiToken);
-        deleteDataset.prettyPrint();
-        deleteDataset.then().assertThat()
-                .statusCode(200);
+                .statusCode(200)
+                .body("data.files[0].dataFile.storageIdentifier", startsWith(driverId + "://"));
 
-        Response deleteDataverse = UtilIT.deleteDataverse(dataverseAlias, apiToken);
-        deleteDataverse.prettyPrint();
-        deleteDataverse.then().assertThat()
-                .statusCode(200);
-        
-        Response deleteUser = UtilIT.deleteUser(username);
-        deleteUser.prettyPrint();
-        deleteUser.then().assertThat()
-                .statusCode(200);
+        String fileId = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.id");
+
+        Response getfileMetadata = UtilIT.getFileData(fileId, apiToken);
+        getfileMetadata.prettyPrint();
+        getfileMetadata.then().assertThat().statusCode(200);
+
+        String storageIdentifier = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.storageIdentifier");
+        String keyInDataverse = storageIdentifier.split(":")[2];
+        Assertions.assertEquals(driverId + "://" + bucketName + ":" + keyInDataverse, storageIdentifier);
+
+        for (Bucket bucket : s3.listBuckets()) {
+            System.out.println("bucket: " + bucket);
+        }
+
+//        List<S3ObjectSummary> summaries = s3.listObjects(bucketName).getObjectSummaries();
+//        for (S3ObjectSummary summary : summaries) {
+//            System.out.println("summary: " + summary);
+//            /**
+//             * summary: S3ObjectSummary{bucketName='mybucket',
+//             * key='10.5072/FK2/6MGSJD/18b631645ef-4c6a6c2d49f8',
+//             * eTag='60b725f10c9c85c70d97880dfe8191b3', size=2, lastModified=Tue
+//             * Oct 24 19:08:06 UTC 2023, storageClass='STANDARD', owner=S3Owner
+//             * [name=webfile,id=75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a]}
+//             */
+//        }
+        String keyInS3 = datasetStorageIdentifier + "/" + keyInDataverse;
+        String s3Object = s3.getObjectAsString(bucketName, keyInS3);
+        System.out.println("s3Object: " + s3Object);
+
+        // The file uploaded above only contains the character "a".
+        assertEquals("a".trim(), s3Object.trim());
+
+        Response deleteFile = UtilIT.deleteFileApi(Integer.parseInt(fileId), apiToken);
+        deleteFile.prettyPrint();
+        deleteFile.then().assertThat().statusCode(200);
+
+        AmazonS3Exception expectedException = null;
+        try {
+            s3.getObjectAsString(bucketName, keyInS3);
+        } catch (AmazonS3Exception ex) {
+            expectedException = ex;
+        }
+        assertNotNull(expectedException);
+        // 404 because the file has been sucessfully deleted
+        assertEquals(404, expectedException.getStatusCode());
+
+    }
+
+    //TODO: move these into UtilIT. They are here for now to avoid merge conflicts
+    static Response listStorageDrivers(String apiToken) {
+        return given()
+                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/admin/dataverse/storageDrivers");
     }
+
+    static Response getStorageDriver(String dvAlias, String apiToken) {
+        return given()
+                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/admin/dataverse/" + dvAlias + "/storageDriver");
+    }
+
+    static Response setStorageDriver(String dvAlias, String label, String apiToken) {
+        return given()
+                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
+                .body(label)
+                .put("/api/admin/dataverse/" + dvAlias + "/storageDriver");
+    }
+
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIOLocalstackIT.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIOLocalstackIT.java
new file mode 100644
index 00000000000..140b558fc1d
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIOLocalstackIT.java
@@ -0,0 +1,153 @@
+package edu.harvard.iq.dataverse.dataaccess;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.util.FileUtil;
+import edu.harvard.iq.dataverse.util.testing.Tags;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Scanner;
+import java.util.UUID;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.testcontainers.containers.localstack.LocalStackContainer;
+import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+// https://java.testcontainers.org/modules/localstack/
+@Tag(Tags.INTEGRATION_TEST)
+@Tag(Tags.USES_TESTCONTAINERS)
+@Testcontainers(disabledWithoutDocker = true)
+@ExtendWith(MockitoExtension.class)
+class S3AccessIOLocalstackIT {
+
+    @BeforeAll
+    static void setUp() {
+        System.setProperty(staticFiles + "access-key", localstack.getAccessKey());
+        System.setProperty(staticFiles + "secret-key", localstack.getSecretKey());
+        System.setProperty(staticFiles + "custom-endpoint-url", localstack.getEndpoint().toString());
+        System.setProperty(staticFiles + "custom-endpoint-region", localstack.getRegion());
+        System.setProperty(staticFiles + "bucket-name", bucketName);
+
+        s3 = AmazonS3ClientBuilder
+                .standard()
+                .withEndpointConfiguration(
+                        new AwsClientBuilder.EndpointConfiguration(
+                                localstack.getEndpoint().toString(),
+                                localstack.getRegion()
+                        )
+                )
+                .withCredentials(
+                        new AWSStaticCredentialsProvider(
+                                new BasicAWSCredentials(localstack.getAccessKey(), localstack.getSecretKey())
+                        )
+                )
+                .build();
+        s3.createBucket(bucketName);
+    }
+
+    static final String storageDriverId = "si1";
+    static final String staticFiles = "dataverse.files." + storageDriverId + ".";
+    static final String bucketName = "bucket-" + UUID.randomUUID().toString();
+    static AmazonS3 s3 = null;
+
+    static DockerImageName localstackImage = DockerImageName.parse("localstack/localstack:2.3.2");
+    @Container
+    static LocalStackContainer localstack = new LocalStackContainer(localstackImage)
+            .withServices(S3);
+
+    //new S3AccessIO<>(dvObject, req, storageDriverId);
+    @Test
+    void test1() {
+        DvObject dvObject = new Dataset();
+        dvObject.setProtocol("doi");
+        dvObject.setAuthority("10.5072/FK2");
+        dvObject.setIdentifier("ABC123");
+        DataAccessRequest req = null;
+        S3AccessIO s3AccessIO = new S3AccessIO<>(dvObject, req, storageDriverId);
+        String textIn = "Hello";
+        InputStream inputStream = new ByteArrayInputStream(textIn.getBytes());
+        // Without this temp directory, saveInputStream fails
+        String tempDirPath = "/tmp/dataverse/temp";
+        try {
+            Files.createDirectories(Paths.get(tempDirPath));
+        } catch (IOException ex) {
+            System.out.println("failed to create " + tempDirPath + ": " + ex);
+        }
+        try {
+            s3AccessIO.saveInputStream(inputStream);
+            System.out.println("save complete!");
+        } catch (IOException ex) {
+            System.out.println("saveInputStream exception: " + ex);
+        }
+
+        String textOut = null;
+        try {
+            textOut = new Scanner(s3AccessIO.getInputStream()).useDelimiter("\\A").next();
+        } catch (IOException ex) {
+        }
+        assertEquals(textIn, textOut);
+    }
+
+    // testing a specific constructor
+    @Test
+    void test2() {
+        Dataset dataset = new Dataset();
+        dataset.setProtocol("doi");
+        dataset.setAuthority("10.5072/FK2");
+        dataset.setIdentifier("ABC123");
+        String sid = sid = bucketName + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + FileUtil.generateStorageIdentifier();
+        S3AccessIO<DataFile> s3io = new S3AccessIO<DataFile>(sid, storageDriverId);
+    }
+
+    // just to test this: saveInputStream exception: java.io.IOException: ERROR: s3 not initialised
+    @Test
+    void test3() {
+        DvObject dvObject = new Dataset();
+        dvObject.setProtocol("doi");
+        dvObject.setAuthority("10.5072/FK2");
+        dvObject.setIdentifier("ABC123");
+        DataAccessRequest req = null;
+        AmazonS3 nullAmazonS3 = null;
+        S3AccessIO s3AccessIO = new S3AccessIO<>(dvObject, req, nullAmazonS3, storageDriverId);
+        InputStream inputStream = null;
+        try {
+            s3AccessIO.saveInputStream(inputStream);
+            System.out.println("save complete!");
+        } catch (IOException ex) {
+            System.out.println("saveInputStream exception: " + ex);
+        }
+    }
+
+    @Test
+    void test4() {
+        DvObject dvObject = new DataFile();
+        dvObject.setProtocol("doi");
+        dvObject.setAuthority("10.5072/FK2");
+        dvObject.setIdentifier("ABC123");
+        DataAccessRequest req = null;
+        S3AccessIO s3AccessIO = new S3AccessIO<>(dvObject, req, storageDriverId);
+        InputStream inputStream = null;
+        try {
+            s3AccessIO.saveInputStream(inputStream);
+            System.out.println("save complete!");
+        } catch (IOException ex) {
+            System.out.println("saveInputStream exception: " + ex);
+        }
+    }
+}

From 53af167b5d746544fe3ebdb3eb16656b7ed2fa76 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 31 Oct 2023 14:23:30 -0400
Subject: [PATCH 750/922] stub out REST Assured code for testing direct upload
 #6783

---
 .../iq/dataverse/api/S3AccessDirectIT.java    | 97 +++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/api/S3AccessDirectIT.java

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessDirectIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessDirectIT.java
new file mode 100644
index 00000000000..1e44d952af7
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessDirectIT.java
@@ -0,0 +1,97 @@
+package edu.harvard.iq.dataverse.api;
+
+import io.restassured.RestAssured;
+import static io.restassured.RestAssured.given;
+import io.restassured.path.json.JsonPath;
+import io.restassured.response.Response;
+import io.restassured.specification.RequestSpecification;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.junit.jupiter.api.Test;
+
+public class S3AccessDirectIT {
+
+    @Test
+    public void testS3DirectUpload() {
+        // TODO: remove all these constants
+        RestAssured.baseURI = "https://demo.dataverse.org";
+        String apiToken = "";
+        String datasetPid = "doi:10.70122/FK2/UBWSJU";
+        String datasetId = "2106131";
+        long size = 1000000000l;
+
+        Response getUploadUrls = getUploadUrls(datasetPid, size, apiToken);
+        getUploadUrls.prettyPrint();
+        getUploadUrls.then().assertThat().statusCode(200);
+
+        String url = JsonPath.from(getUploadUrls.asString()).getString("data.url");
+        String partSize = JsonPath.from(getUploadUrls.asString()).getString("data.partSize");
+        String storageIdentifier = JsonPath.from(getUploadUrls.asString()).getString("data.storageIdentifier");
+        System.out.println("url: " + url);
+        System.out.println("partSize: " + partSize);
+        System.out.println("storageIdentifier: " + storageIdentifier);
+
+        System.out.println("uploading file via direct upload");
+        String decodedUrl = null;
+        try {
+            decodedUrl = URLDecoder.decode(url, StandardCharsets.UTF_8.name());
+        } catch (UnsupportedEncodingException ex) {
+        }
+
+        InputStream inputStream = new ByteArrayInputStream("bumble".getBytes(StandardCharsets.UTF_8));
+        Response uploadFileDirect = uploadFileDirect(decodedUrl, inputStream);
+        uploadFileDirect.prettyPrint();
+        uploadFileDirect.then().assertThat().statusCode(200);
+
+        // TODO: Use MD5 or whatever Dataverse is configured for and
+        // actually calculate it.
+        String jsonData = """
+{
+    "description": "My description.",
+    "directoryLabel": "data/subdir1",
+    "categories": [
+      "Data"
+    ],
+    "restrict": "false",
+    "storageIdentifier": "%s",
+    "fileName": "file1.txt",
+    "mimeType": "text/plain",
+    "checksum": {
+      "@type": "SHA-1",
+      "@value": "123456"
+    }
+}
+""".formatted(storageIdentifier);
+        Response addRemoteFile = UtilIT.addRemoteFile(datasetId, jsonData, apiToken);
+        addRemoteFile.prettyPrint();
+        addRemoteFile.then().assertThat()
+                .statusCode(200);
+    }
+
+    static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes, String apiToken) {
+        String idInPath = idOrPersistentIdOfDataset; // Assume it's a number.
+        String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
+        if (!NumberUtils.isCreatable(idOrPersistentIdOfDataset)) {
+            idInPath = ":persistentId";
+            optionalQueryParam = "&persistentId=" + idOrPersistentIdOfDataset;
+        }
+        RequestSpecification requestSpecification = given();
+        if (apiToken != null) {
+            requestSpecification = given()
+                    .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
+        }
+        return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam);
+    }
+
+    static Response uploadFileDirect(String url, InputStream inputStream) {
+        return given()
+                .header("x-amz-tagging", "dv-state=temp")
+                .body(inputStream)
+                .put(url);
+    }
+
+}

From 69899c2f1445f73e36ad1f57ca196fb07fe161b0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 31 Oct 2023 14:24:27 -0400
Subject: [PATCH 751/922] explain that 200 OK is expected from direct upload
 #6783

---
 doc/sphinx-guides/source/developers/s3-direct-upload-api.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
index 423fb02d385..d7f270a4e38 100644
--- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
+++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
@@ -69,8 +69,9 @@ In the single part case, only one call to the supplied URL is required:
 
 .. code-block:: bash
 
-    curl -H 'x-amz-tagging:dv-state=temp' -X PUT -T <filename> "<supplied url>"
+    curl -i -H 'x-amz-tagging:dv-state=temp' -X PUT -T <filename> "<supplied url>"
 
+Note that without the ``-i`` flag, you should not expect any output from the command above. With the ``-i`` flag, you should expect to see a "200 OK" response.
 
 In the multipart case, the client must send each part and collect the 'eTag' responses from the server. The calls for this are the same as the one for the single part case except that each call should send a <partSize> slice of the total file, with the last part containing the remaining bytes.
 The responses from the S3 server for these calls will include the 'eTag' for the uploaded part. 

From bbf6789b104d42970a251e09bb967e09bd908bd7 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 1 Nov 2023 15:03:43 -0400
Subject: [PATCH 752/922] refactor and add direct upload test #6783

---
 docker-compose-dev.yml                        |   2 +-
 .../harvard/iq/dataverse/api/S3AccessIT.java  | 321 +++++++++++++-----
 2 files changed, 245 insertions(+), 78 deletions(-)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 769c24fb3a5..73c00fb02c9 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -29,7 +29,7 @@ services:
         -Ddataverse.files.localstack1.custom-endpoint-region=us-east-2
         -Ddataverse.files.localstack1.bucket-name=mybucket
         -Ddataverse.files.localstack1.path-style-access=true
-        -Ddataverse.files.localstack1.upload-redirect=false
+        -Ddataverse.files.localstack1.upload-redirect=true
         -Ddataverse.files.localstack1.access-key=default
         -Ddataverse.files.localstack1.secret-key=default
         -Ddataverse.files.minio1.type=s3
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index 891a54e87c6..9173775875b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -9,73 +9,65 @@
 import com.amazonaws.services.s3.model.AmazonS3Exception;
 import com.amazonaws.services.s3.model.Bucket;
 import com.amazonaws.services.s3.model.HeadBucketRequest;
-import com.amazonaws.services.s3.model.HeadBucketResult;
 import io.restassured.RestAssured;
 import static io.restassured.RestAssured.given;
 import io.restassured.path.json.JsonPath;
 import io.restassured.response.Response;
-import java.util.List;
+import io.restassured.specification.RequestSpecification;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
 import java.util.logging.Logger;
+import org.apache.commons.lang3.math.NumberUtils;
 import static org.hamcrest.CoreMatchers.equalTo;
-
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
-
 import static org.hamcrest.Matchers.startsWith;
 import org.junit.jupiter.api.Assertions;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 
 /**
- * This test requires services spun up in Docker.
+ * This test requires LocalStack and Minio to be running. Developers can use our
+ * docker-compose file, which has all the necessary configuration.
  */
 public class S3AccessIT {
 
     private static final Logger logger = Logger.getLogger(S3AccessIT.class.getCanonicalName());
 
-    public enum TypesOfS3 {
-        MINIO,
-        LOCALSTACK
-    };
-
-    static final String accessKey = "minioadmin";
-    static final String secretKey = "minioadmin";
-    static final String bucketName = "mybucket";
-    static String driverId;
-    static String driverLabel;
-    static AmazonS3 s3 = null;
+    static final String BUCKET_NAME = "mybucket";
+    static AmazonS3 s3localstack = null;
+    static AmazonS3 s3minio = null;
 
     @BeforeAll
     public static void setUp() {
         RestAssured.baseURI = UtilIT.getRestAssuredBaseUri();
 
-        TypesOfS3 typeToTest = TypesOfS3.LOCALSTACK;
-        typeToTest = TypesOfS3.MINIO;
-
-        switch (typeToTest) {
-            case LOCALSTACK -> {
-                driverId = "localstack1";
-                driverLabel = "LocalStack";
-                s3 = AmazonS3ClientBuilder.standard()
-                        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
-                        .withEndpointConfiguration(new EndpointConfiguration("s3.localhost.localstack.cloud:4566", Regions.US_EAST_2.getName())).build();
-            }
-            case MINIO -> {
-                driverId = "minio1";
-                driverLabel = "MinIO";
-                s3 = AmazonS3ClientBuilder.standard()
-                        // https://stackoverflow.com/questions/72205086/amazonss3client-throws-unknownhostexception-if-attempting-to-connect-to-a-local
-                        .withPathStyleAccessEnabled(Boolean.TRUE)
-                        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)))
-                        .withEndpointConfiguration(new EndpointConfiguration("http://localhost:9000", Regions.US_EAST_1.getName())).build();
-//                String location = s3.getBucketLocation(bucketName);
-//                if (location != "US") {
-//                    Bucket bucket = s3.createBucket(bucketName);
-//                }
-            }
+        // At least in when spun up by our docker-compose file, the creds don't matter for LocalStack.
+        String accessKeyLocalStack = "whatever";
+        String secretKeyLocalStack = "not used";
+
+        s3localstack = AmazonS3ClientBuilder.standard()
+                .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyLocalStack, secretKeyLocalStack)))
+                .withEndpointConfiguration(new EndpointConfiguration("s3.localhost.localstack.cloud:4566", Regions.US_EAST_2.getName())).build();
+
+        String accessKeyMinio = "minioadmin";
+        String secretKeyMinio = "minioadmin";
+        s3minio = AmazonS3ClientBuilder.standard()
+                // https://stackoverflow.com/questions/72205086/amazonss3client-throws-unknownhostexception-if-attempting-to-connect-to-a-local
+                .withPathStyleAccessEnabled(Boolean.TRUE)
+                .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyMinio, secretKeyMinio)))
+                .withEndpointConfiguration(new EndpointConfiguration("http://localhost:9000", Regions.US_EAST_1.getName())).build();
+
+        System.out.println("buckets on LocalStack before attempting to create " + BUCKET_NAME);
+        for (Bucket bucket : s3localstack.listBuckets()) {
+            System.out.println("bucket: " + bucket);
         }
-        System.out.println("buckets before attempting to create " + bucketName);
-        for (Bucket bucket : s3.listBuckets()) {
+
+        System.out.println("buckets on MinIO before attempting to create " + BUCKET_NAME);
+        for (Bucket bucket : s3minio.listBuckets()) {
             System.out.println("bucket: " + bucket);
         }
 
@@ -84,27 +76,27 @@ public static void setUp() {
         // because we haven't figured out how to create it properly in Java.
         // Perhaps it is missing ACLs.
         try {
-            s3.headBucket(new HeadBucketRequest(bucketName));
+            s3localstack.headBucket(new HeadBucketRequest(BUCKET_NAME));
         } catch (AmazonS3Exception ex) {
-            s3.createBucket(bucketName);
+            s3localstack.createBucket(BUCKET_NAME);
+        }
+
+        try {
+            s3minio.headBucket(new HeadBucketRequest(BUCKET_NAME));
+        } catch (AmazonS3Exception ex) {
+            s3minio.createBucket(BUCKET_NAME);
         }
 
-//        String location = s3.getBucketLocation(bucketName);
-////        HeadBucketRequest headBucketRequest;
-//        s3.headBucket(headBucketRequest);
-//        if (location != null && !"US".equals(location)) {
-//            System.out.println("Creating bucket. Location was " + location);
-//            Bucket createdBucket = s3.createBucket(bucketName);
-//            System.out.println("created bucket: " + createdBucket);
-//        }
-//        System.out.println("buckets after creating " + bucketName);
-//        for (Bucket bucket : s3.listBuckets()) {
-//            System.out.println("bucket: " + bucket);
-//        }
     }
 
+    /**
+     * We're using MinIO for testing non-direct upload.
+     */
     @Test
-    public void testAddDataFileS3Prefix() {
+    public void testNonDirectUpload() {
+        String driverId = "minio1";
+        String driverLabel = "MinIO";
+
         Response createSuperuser = UtilIT.createRandomUser();
         String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
         String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
@@ -175,25 +167,10 @@ public void testAddDataFileS3Prefix() {
 
         String storageIdentifier = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.storageIdentifier");
         String keyInDataverse = storageIdentifier.split(":")[2];
-        Assertions.assertEquals(driverId + "://" + bucketName + ":" + keyInDataverse, storageIdentifier);
+        Assertions.assertEquals(driverId + "://" + BUCKET_NAME + ":" + keyInDataverse, storageIdentifier);
 
-        for (Bucket bucket : s3.listBuckets()) {
-            System.out.println("bucket: " + bucket);
-        }
-
-//        List<S3ObjectSummary> summaries = s3.listObjects(bucketName).getObjectSummaries();
-//        for (S3ObjectSummary summary : summaries) {
-//            System.out.println("summary: " + summary);
-//            /**
-//             * summary: S3ObjectSummary{bucketName='mybucket',
-//             * key='10.5072/FK2/6MGSJD/18b631645ef-4c6a6c2d49f8',
-//             * eTag='60b725f10c9c85c70d97880dfe8191b3', size=2, lastModified=Tue
-//             * Oct 24 19:08:06 UTC 2023, storageClass='STANDARD', owner=S3Owner
-//             * [name=webfile,id=75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a]}
-//             */
-//        }
         String keyInS3 = datasetStorageIdentifier + "/" + keyInDataverse;
-        String s3Object = s3.getObjectAsString(bucketName, keyInS3);
+        String s3Object = s3minio.getObjectAsString(BUCKET_NAME, keyInS3);
         System.out.println("s3Object: " + s3Object);
 
         // The file uploaded above only contains the character "a".
@@ -205,7 +182,175 @@ public void testAddDataFileS3Prefix() {
 
         AmazonS3Exception expectedException = null;
         try {
-            s3.getObjectAsString(bucketName, keyInS3);
+            s3minio.getObjectAsString(BUCKET_NAME, keyInS3);
+        } catch (AmazonS3Exception ex) {
+            expectedException = ex;
+        }
+        assertNotNull(expectedException);
+        // 404 because the file has been sucessfully deleted
+        assertEquals(404, expectedException.getStatusCode());
+
+    }
+
+    /**
+     * We use LocalStack to test direct upload.
+     */
+    @Test
+    public void testDirectUpload() {
+        String driverId = "localstack1";
+        String driverLabel = "LocalStack";
+        Response createSuperuser = UtilIT.createRandomUser();
+        String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
+        String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
+        UtilIT.makeSuperUser(superusername);
+        Response storageDrivers = listStorageDrivers(superuserApiToken);
+        storageDrivers.prettyPrint();
+        // TODO where is "Local/local" coming from?
+        String drivers = """
+{
+    "status": "OK",
+    "data": {
+        "LocalStack": "localstack1",
+        "MinIO": "minio1",
+        "Local": "local",
+        "Filesystem": "file1"
+    }
+}""";
+
+        //create user who will make a dataverse/dataset
+        Response createUser = UtilIT.createRandomUser();
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response originalStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        originalStorageDriver.prettyPrint();
+        originalStorageDriver.then().assertThat()
+                .body("data.message", equalTo("undefined"))
+                .statusCode(200);
+
+        Response setStorageDriverToS3 = setStorageDriver(dataverseAlias, driverLabel, superuserApiToken);
+        setStorageDriverToS3.prettyPrint();
+        setStorageDriverToS3.then().assertThat()
+                .statusCode(200);
+
+        Response updatedStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        updatedStorageDriver.prettyPrint();
+        updatedStorageDriver.then().assertThat()
+                .statusCode(200);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.prettyPrint();
+        createDatasetResponse.then().assertThat().statusCode(201);
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        String datasetPid = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId");
+        String datasetStorageIdentifier = datasetPid.substring(4);
+
+        Response getDatasetMetadata = UtilIT.nativeGet(datasetId, apiToken);
+        getDatasetMetadata.prettyPrint();
+        getDatasetMetadata.then().assertThat().statusCode(200);
+
+//        //upload a tabular file via native, check storage id prefix for driverId
+//        String pathToFile = "scripts/search/data/tabular/1char";
+//        Response addFileResponse = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken);
+//        addFileResponse.prettyPrint();
+//        addFileResponse.then().assertThat()
+//                .statusCode(200)
+//                .body("data.files[0].dataFile.storageIdentifier", startsWith(driverId + "://"));
+//
+//        String fileId = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.id");
+        long size = 1000000000l;
+        Response getUploadUrls = getUploadUrls(datasetPid, size, apiToken);
+        getUploadUrls.prettyPrint();
+        getUploadUrls.then().assertThat().statusCode(200);
+
+        String url = JsonPath.from(getUploadUrls.asString()).getString("data.url");
+        String partSize = JsonPath.from(getUploadUrls.asString()).getString("data.partSize");
+        String storageIdentifier = JsonPath.from(getUploadUrls.asString()).getString("data.storageIdentifier");
+        System.out.println("url: " + url);
+        System.out.println("partSize: " + partSize);
+        System.out.println("storageIdentifier: " + storageIdentifier);
+
+        System.out.println("uploading file via direct upload");
+        String decodedUrl = null;
+        try {
+            decodedUrl = URLDecoder.decode(url, StandardCharsets.UTF_8.name());
+        } catch (UnsupportedEncodingException ex) {
+        }
+
+        // change to localhost because LocalStack is running in a container locally
+        String localhostUrl = decodedUrl.replace("http://localstack", "http://localhost");
+        String contentsOfFile = "foobar";
+
+        InputStream inputStream = new ByteArrayInputStream(contentsOfFile.getBytes(StandardCharsets.UTF_8));
+        Response uploadFileDirect = uploadFileDirect(localhostUrl, inputStream);
+        uploadFileDirect.prettyPrint();
+        /*
+        Direct upload to MinIO is failing with errors like this:
+        <Error>
+          <Code>SignatureDoesNotMatch</Code>
+          <Message>The request signature we calculated does not match the signature you provided. Check your key and signing method.</Message>
+          <Key>10.5072/FK2/KGFCEJ/18b8c06688c-21b8320a3ee5</Key>
+          <BucketName>mybucket</BucketName>
+          <Resource>/mybucket/10.5072/FK2/KGFCEJ/18b8c06688c-21b8320a3ee5</Resource>
+          <RequestId>1793915CCC5BC95C</RequestId>
+          <HostId>dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8</HostId>
+        </Error>
+         */
+        uploadFileDirect.then().assertThat().statusCode(200);
+
+        // TODO: Use MD5 or whatever Dataverse is configured for and
+        // actually calculate it.
+        String jsonData = """
+{
+    "description": "My description.",
+    "directoryLabel": "data/subdir1",
+    "categories": [
+      "Data"
+    ],
+    "restrict": "false",
+    "storageIdentifier": "%s",
+    "fileName": "file1.txt",
+    "mimeType": "text/plain",
+    "checksum": {
+      "@type": "SHA-1",
+      "@value": "123456"
+    }
+}
+""".formatted(storageIdentifier);
+
+        // "There was an error when trying to add the new file. File size must be explicitly specified when creating DataFiles with Direct Upload"
+        Response addRemoteFile = UtilIT.addRemoteFile(datasetId.toString(), jsonData, apiToken);
+        addRemoteFile.prettyPrint();
+        addRemoteFile.then().assertThat()
+                .statusCode(200);
+
+        String fileId = JsonPath.from(addRemoteFile.asString()).getString("data.files[0].dataFile.id");
+        Response getfileMetadata = UtilIT.getFileData(fileId, apiToken);
+        getfileMetadata.prettyPrint();
+        getfileMetadata.then().assertThat().statusCode(200);
+
+//        String storageIdentifier = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.storageIdentifier");
+        String keyInDataverse = storageIdentifier.split(":")[2];
+        Assertions.assertEquals(driverId + "://" + BUCKET_NAME + ":" + keyInDataverse, storageIdentifier);
+
+        String keyInS3 = datasetStorageIdentifier + "/" + keyInDataverse;
+        String s3Object = s3localstack.getObjectAsString(BUCKET_NAME, keyInS3);
+        System.out.println("s3Object: " + s3Object);
+
+//        assertEquals(contentsOfFile.trim(), s3Object.trim());
+        assertEquals(contentsOfFile, s3Object);
+
+        Response deleteFile = UtilIT.deleteFileApi(Integer.parseInt(fileId), apiToken);
+        deleteFile.prettyPrint();
+        deleteFile.then().assertThat().statusCode(200);
+
+        AmazonS3Exception expectedException = null;
+        try {
+            s3localstack.getObjectAsString(BUCKET_NAME, keyInS3);
         } catch (AmazonS3Exception ex) {
             expectedException = ex;
         }
@@ -235,4 +380,26 @@ static Response setStorageDriver(String dvAlias, String label, String apiToken)
                 .put("/api/admin/dataverse/" + dvAlias + "/storageDriver");
     }
 
+    static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes, String apiToken) {
+        String idInPath = idOrPersistentIdOfDataset; // Assume it's a number.
+        String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
+        if (!NumberUtils.isCreatable(idOrPersistentIdOfDataset)) {
+            idInPath = ":persistentId";
+            optionalQueryParam = "&persistentId=" + idOrPersistentIdOfDataset;
+        }
+        RequestSpecification requestSpecification = given();
+        if (apiToken != null) {
+            requestSpecification = given()
+                    .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
+        }
+        return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam);
+    }
+
+    static Response uploadFileDirect(String url, InputStream inputStream) {
+        return given()
+                .header("x-amz-tagging", "dv-state=temp")
+                .body(inputStream)
+                .put(url);
+    }
+
 }

From 20bbc700d39c6e49ab3d2914f1982ef047c9760c Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 8 Nov 2023 10:24:00 -0500
Subject: [PATCH 753/922] add tests for direct (and non-direct) download #6783

---
 docker-compose-dev.yml                        |  2 +
 .../harvard/iq/dataverse/api/S3AccessIT.java  | 39 +++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 73c00fb02c9..6bc50f7e764 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -30,6 +30,7 @@ services:
         -Ddataverse.files.localstack1.bucket-name=mybucket
         -Ddataverse.files.localstack1.path-style-access=true
         -Ddataverse.files.localstack1.upload-redirect=true
+        -Ddataverse.files.localstack1.download-redirect=true
         -Ddataverse.files.localstack1.access-key=default
         -Ddataverse.files.localstack1.secret-key=default
         -Ddataverse.files.minio1.type=s3
@@ -39,6 +40,7 @@ services:
         -Ddataverse.files.minio1.bucket-name=mybucket
         -Ddataverse.files.minio1.path-style-access=true
         -Ddataverse.files.minio1.upload-redirect=false
+        -Ddataverse.files.minio1.download-redirect=false
         -Ddataverse.files.minio1.access-key=minioadmin
         -Ddataverse.files.minio1.secret-key=minioadmin
     ports:
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index 9173775875b..f5e4ce6a794 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -11,6 +11,7 @@
 import com.amazonaws.services.s3.model.HeadBucketRequest;
 import io.restassured.RestAssured;
 import static io.restassured.RestAssured.given;
+import io.restassured.http.Header;
 import io.restassured.path.json.JsonPath;
 import io.restassured.response.Response;
 import io.restassured.specification.RequestSpecification;
@@ -176,6 +177,13 @@ public void testNonDirectUpload() {
         // The file uploaded above only contains the character "a".
         assertEquals("a".trim(), s3Object.trim());
 
+        System.out.println("non-direct download...");
+        Response downloadFile = UtilIT.downloadFile(Integer.valueOf(fileId), apiToken);
+        downloadFile.then().assertThat().statusCode(200);
+
+        String contentsOfDownloadedFile = downloadFile.getBody().asString();
+        assertEquals("a\n", contentsOfDownloadedFile);
+
         Response deleteFile = UtilIT.deleteFileApi(Integer.parseInt(fileId), apiToken);
         deleteFile.prettyPrint();
         deleteFile.then().assertThat().statusCode(200);
@@ -344,6 +352,28 @@ public void testDirectUpload() {
 //        assertEquals(contentsOfFile.trim(), s3Object.trim());
         assertEquals(contentsOfFile, s3Object);
 
+        System.out.println("direct download...");
+        Response getHeaders = downloadFileNoRedirect(Integer.valueOf(fileId), apiToken);
+        for (Header header : getHeaders.getHeaders()) {
+            System.out.println("direct download header: " + header);
+        }
+        getHeaders.then().assertThat().statusCode(303);
+
+        String urlFromResponse = getHeaders.getHeader("Location");
+        String localhostDownloadUrl = urlFromResponse.replace("localstack", "localhost");
+        String decodedDownloadUrl = null;
+        try {
+            decodedDownloadUrl = URLDecoder.decode(localhostDownloadUrl, StandardCharsets.UTF_8.name());
+        } catch (UnsupportedEncodingException ex) {
+        }
+
+        Response downloadFile = downloadFromUrl(decodedDownloadUrl);
+        downloadFile.prettyPrint();
+        downloadFile.then().assertThat().statusCode(200);
+
+        String contentsOfDownloadedFile = downloadFile.getBody().asString();
+        assertEquals(contentsOfFile, contentsOfDownloadedFile);
+
         Response deleteFile = UtilIT.deleteFileApi(Integer.parseInt(fileId), apiToken);
         deleteFile.prettyPrint();
         deleteFile.then().assertThat().statusCode(200);
@@ -402,4 +432,13 @@ static Response uploadFileDirect(String url, InputStream inputStream) {
                 .put(url);
     }
 
+    static Response downloadFileNoRedirect(Integer fileId, String apiToken) {
+        return given().when().redirects().follow(false)
+                .get("/api/access/datafile/" + fileId + "?key=" + apiToken);
+    }
+
+    static Response downloadFromUrl(String url) {
+        return given().get(url);
+    }
+
 }

From 6608dc27a81a53343a4ea95a88e348a36545a06d Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 9 Nov 2023 13:14:48 -0500
Subject: [PATCH 754/922] tweak release note #6783

---
 doc/release-notes/6783-s3-tests.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/release-notes/6783-s3-tests.md b/doc/release-notes/6783-s3-tests.md
index 1febb87aaed..1b9bb400cc6 100644
--- a/doc/release-notes/6783-s3-tests.md
+++ b/doc/release-notes/6783-s3-tests.md
@@ -1,3 +1,3 @@
-Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. See S3AccessIT which executes API (end to end) tests.
+Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT.
 
-In addition, a new integration test test class (not an API test, the new kind launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.
+In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.

From 3d94c8f747533f770a11c85e839533caf53bd7c5 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 9 Nov 2023 13:49:39 -0500
Subject: [PATCH 755/922] mention S3 in testing docs, other cleanup #6783

---
 .../source/developers/testing.rst             | 41 ++++++++++---------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index abecaa09fad..fd0f227342f 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -190,42 +190,38 @@ Finally, run the script:
 
   $ ./ec2-create-instance.sh -g jenkins.yml -l log_dir
 
-Running the full API test suite using Docker
+Running the Full API Test Suite Using Docker
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To run the full suite of integration tests on your laptop, running Dataverse and its dependencies in Docker, as explained in the :doc:`/container/dev-usage` section of the Container Guide.
+To run the full suite of integration tests on your laptop, we recommend running Dataverse and its dependencies in Docker, as explained in the :doc:`/container/dev-usage` section of the Container Guide. This environment provides additional services (such as S3) that are used in testing.
 
-Alternatively, you can run tests against the app server running on your laptop by following the "getting set up" steps below.
+Running the APIs Without Docker (Classic Dev Env)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Getting Set Up to Run REST Assured Tests
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+While it is possible to run a good number of API tests without using Docker in our :doc:`classic-dev-env`, we are transitioning toward including additional services (such as S3) in our Dockerized development environment (:doc:`/container/dev-usage`), so you will probably find it more convenient to it instead.
 
-Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, the Dataverse Software needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of the Dataverse Software, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment.
-
-The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put the Dataverse Software in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production.
-
-The instructions below assume a relatively static dev environment on a Mac. There is a newer "all in one" Docker-based approach documented in the :doc:`/developers/containers` section under "Docker" that you may like to play with as well.
+Unit tests are run automatically on every build, but dev environments and servers require special setup to run API (REST Assured) tests. In short, the Dataverse software needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed (this is done automatically in the Dockerized environment, as well as the steps described below). This differs greatly from the out-of-the-box behavior of the Dataverse software, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment.
 
 The Burrito Key
 ^^^^^^^^^^^^^^^
 
-For reasons that have been lost to the mists of time, the Dataverse Software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database):
+For reasons that have been lost to the mists of time, the Dataverse software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database):
 
 ``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/BuiltinUsers.KEY``
 
-Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as Dataverse collections, datasets, and files.
+Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as collections, datasets, and files.
 
-Root Dataverse Collection Permissions
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Root Collection Permissions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root Dataverse collection. Navigate to Permissions, then the Edit Access button. Under "Who can add to this Dataverse collection?" choose "Anyone with a Dataverse installation account can add sub Dataverse collections and datasets" if it isn't set to this already.
+In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root collection. Navigate to Permissions, then the Edit Access button. Under "Who can add to this collection?" choose "Anyone with a Dataverse installation account can add sub collections and datasets" if it isn't set to this already.
 
 Alternatively, this same step can be done with this script: ``scripts/search/tests/grant-authusers-add-on-root``
 
-Publish Root Dataverse Collection
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Publish Root Collection
+^^^^^^^^^^^^^^^^^^^^^^^
 
-The root Dataverse collection must be published for some of the REST Assured tests to run.
+The root collection must be published for some of the REST Assured tests to run.
 
 dataverse.siteUrl
 ^^^^^^^^^^^^^^^^^
@@ -274,15 +270,20 @@ Remember, it’s only a test (and it's not graded)! Some guidelines to bear in m
 - Assert the conditions of success / return values for each operation
   * A useful resource would be `HTTP status codes <https://www.restapitutorial.com/httpstatuscodes.html>`_
 - Let the code do the labor; automate everything that happens when you run your test file.
+- If you need to test an optional service (S3, etc.), add it to our docker compose file. See :doc:`/container/dev-usage`.
 - Just as with any development, if you’re stuck: ask for help!
 
 To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven <https://maven.apache.org/ref/3.1.0/maven-embedder/cli.html>`_. You should have Maven installed as per the `Development Environment <https://guides.dataverse.org/en/latest/developers/dev-environment.html>`_ guide, but if not it’s easily done via Homebrew: ``brew install maven``.
 
 Once installed, you may run commands with ``mvn [options] [<goal(s)>] [<phase(s)>]``.
 
-+ If you want to run just one particular API test, it’s as easy as you think:
++ If you want to run just one particular API test class:
+
+  ``mvn test -Dtest=UsersIT``
+
++ If you want to run just one particular API test method,
 
-  ``mvn test -Dtest=FileRecordJobIT``
+  ``mvn test -Dtest=UsersIT#testMergeAccounts``
 
 + To run more than one test at a time, separate by commas:
 

From d4bab0763d770d14bfd57a2f90250d11cc153b7c Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 14 Nov 2023 08:18:00 -0500
Subject: [PATCH 756/922] add S3AccessIT to the list of integration-tests #6783

---
 tests/integration-tests.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration-tests.txt b/tests/integration-tests.txt
index bb3bc7f9ce6..3c4f7dce31f 100644
--- a/tests/integration-tests.txt
+++ b/tests/integration-tests.txt
@@ -1 +1 @@
-DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,ProvIT
+DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,HarvestingClientsIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT,AuxiliaryFilesIT,InvalidCharactersIT,LicensesIT,NotificationsIT,BagIT,MetadataBlocksIT,NetcdfIT,SignpostingIT,FitsIT,LogoutIT,ProvIT,S3AccessIT

From ff722a257c7196e29884666b2875f0ee667940c2 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 20 Nov 2023 10:55:02 -0500
Subject: [PATCH 757/922] descriptive labels for file stores #9622

---
 .../source/installation/config.rst            | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 13a7367de44..cabff6215ce 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -539,6 +539,27 @@ If you wish to change which store is used by default, you'll need to delete the
 
 It is also possible to set maximum file upload size limits per store. See the :ref:`:MaxFileUploadSizeInBytes` setting below.
 
+.. _labels-file-stores:
+
+Labels for File Stores
+++++++++++++++++++++++
+
+If you find yourself adding many file stores with various configurations such as per-file limits and direct upload, you might find it helpful to make the label descriptive.
+
+For example, instead of simply labeling an S3 store as "S3"...
+
+.. code-block:: none
+
+    ./asadmin create-jvm-options "\-Ddataverse.files.s3xl.label=S3"
+
+... you might want to include some extra information such as the example below.
+
+.. code-block:: none
+
+    ./asadmin create-jvm-options "\-Ddataverse.files.s3xl.label=S3XL, Filesize limit: 100GB, direct-upload"
+
+Please keep in mind that the UI will only show so many characters, so labels are best kept short.
+
 .. _storage-files-dir:
 
 File Storage

From 766c9c3da73a02fefa9c465c26e452cbd0d79f62 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 20 Nov 2023 13:32:25 -0500
Subject: [PATCH 758/922] #9464 add base schema as a file

---
 .../iq/dataverse/baseDatasetSchema.json       | 123 ++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json

diff --git a/src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json b/src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json
new file mode 100644
index 00000000000..a37e216b2ea
--- /dev/null
+++ b/src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json
@@ -0,0 +1,123 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$defs": {
+    "field": {
+        "type": "object",
+        "required": ["typeClass", "multiple", "typeName"],
+        "properties": {
+            "value": {
+                "anyOf": [
+                    {
+                        "type": "array"
+                    },
+                    {
+                        "type": "string"
+                    },
+                    {
+                        "$ref": "#/$defs/field"
+                    }
+                ]
+            },
+            "typeClass": {
+                "type": "string"
+            },
+            "multiple": {
+                "type": "boolean"
+            },
+            "typeName": {
+                "type": "string"
+            }
+        }
+    }
+},
+"type": "object",
+"properties": {
+    "datasetVersion": {
+        "type": "object",
+        "properties": {
+           "license": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "uri": {
+                        "type": "string",
+                        "format": "uri"
+                   }
+                },
+                "required": ["name", "uri"]
+            },
+            "metadataBlocks": {
+                "type": "object",
+                "properties": {
+                           "citation": {
+                            "type": "object",
+                            "properties": {
+                                "fields": {
+                                    "type": "array",
+                                    "items": {
+                                        "$ref": "#/$defs/field"
+                                    },
+                                    "minItems": 5,
+                                    "allOf": [
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "title"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "author"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "datasetContact"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "dsDescription"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "subject"
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    ]
+                                }
+                            },
+                            "required": ["fields"]
+                        }
+                    },
+                    "required": ["citation"]
+                }
+            },
+            "required": ["metadataBlocks"]
+        }
+    },
+    "required": ["datasetVersion"]
+}
+

From d34e92711a118b43b90023b458116dba651fe8b0 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Mon, 20 Nov 2023 15:58:06 -0500
Subject: [PATCH 759/922] Remove the authentication annotation from info API

---
 .../harvard/iq/dataverse/api/AbstractApiBean.java | 10 ++++++++++
 .../java/edu/harvard/iq/dataverse/api/Info.java   | 15 ++++++---------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
index 027f9e0fcb1..754ea95e427 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
@@ -636,6 +636,16 @@ protected Response response(DataverseRequestHandler hdl, User user) {
         }
     }
 
+    /***
+     * The preferred way of handling a request from an open API.
+     *
+     * @param hdl handling code block.
+     * @return HTTP Response appropriate for the way {@code hdl} executed.
+     */
+    protected Response response(DataverseRequestHandler hdl) {
+        return response(hdl, null);
+    }
+
     private Response handleDataverseRequestHandlerException(Exception ex) {
         String incidentId = UUID.randomUUID().toString();
         logger.log(Level.SEVERE, "API internal error " + incidentId +": " + ex.getMessage(), ex);
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index 0652539b595..0a3887cbcb3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -35,30 +35,27 @@ public Response getMaxEmbargoDurationInMonths() {
     }
 
     @GET
-    @AuthRequired
     @Path("version")
-    public Response getInfo(@Context ContainerRequestContext crc) {
+    public Response getInfo() {
         String versionStr = systemConfig.getVersion(true);
         String[] comps = versionStr.split("build",2);
         String version = comps[0].trim();
         JsonValue build = comps.length > 1 ? Json.createArrayBuilder().add(comps[1].trim()).build().get(0) : JsonValue.NULL;
 
         return response( req -> ok( Json.createObjectBuilder().add("version", version)
-                                                              .add("build", build)), getRequestUser(crc));
+                                                              .add("build", build)));
     }
 
     @GET
-    @AuthRequired
     @Path("server")
-    public Response getServer(@Context ContainerRequestContext crc) {
-        return response( req -> ok(JvmSettings.FQDN.lookup()), getRequestUser(crc));
+    public Response getServer() {
+        return response( req -> ok(JvmSettings.FQDN.lookup()));
     }
 
     @GET
-    @AuthRequired
     @Path("apiTermsOfUse")
-    public Response getTermsOfUse(@Context ContainerRequestContext crc) {
-        return response( req -> ok(systemConfig.getApiTermsOfUse()), getRequestUser(crc));
+    public Response getTermsOfUse() {
+        return response( req -> ok(systemConfig.getApiTermsOfUse()));
     }
 
     @GET

From 9a5f523a36b3e3aa708628a8de8fcb1bbe52565d Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Mon, 20 Nov 2023 16:03:00 -0500
Subject: [PATCH 760/922] Remove unused libraries

---
 src/main/java/edu/harvard/iq/dataverse/api/Info.java | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index 0a3887cbcb3..5478c3be0bd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -1,6 +1,5 @@
 package edu.harvard.iq.dataverse.api;
 
-import edu.harvard.iq.dataverse.api.auth.AuthRequired;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -9,8 +8,6 @@
 import jakarta.json.JsonValue;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.Path;
-import jakarta.ws.rs.container.ContainerRequestContext;
-import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.Response;
 
 @Path("info")

From c5cbe939f17fff244a88cd1adac6c0aaead7e309 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 20 Nov 2023 16:21:15 -0500
Subject: [PATCH 761/922] add performance section to dev guide #9919

Also cross-link related content.
---
 doc/sphinx-guides/source/admin/monitoring.rst |   2 +-
 doc/sphinx-guides/source/developers/index.rst |   1 +
 .../source/developers/performance.rst         | 196 ++++++++++++++++++
 .../source/developers/testing.rst             |   2 +
 doc/sphinx-guides/source/developers/tips.rst  |   2 +
 .../source/installation/advanced.rst          |   2 +
 6 files changed, 204 insertions(+), 1 deletion(-)
 create mode 100644 doc/sphinx-guides/source/developers/performance.rst

diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst
index e902d5fdcc9..04fba23a3e8 100644
--- a/doc/sphinx-guides/source/admin/monitoring.rst
+++ b/doc/sphinx-guides/source/admin/monitoring.rst
@@ -1,7 +1,7 @@
 Monitoring
 ===========
 
-Once you're in production, you'll want to set up some monitoring. This page may serve as a starting point for you but you are encouraged to share your ideas with the Dataverse community!
+Once you're in production, you'll want to set up some monitoring. This page may serve as a starting point for you but you are encouraged to share your ideas with the Dataverse community! You may also be interested in the :doc:`/developers/performance` section of the Developer Guide.
 
 .. contents:: Contents:
 	:local:
diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst
index 60d97feeef9..141b19d2f89 100755
--- a/doc/sphinx-guides/source/developers/index.rst
+++ b/doc/sphinx-guides/source/developers/index.rst
@@ -21,6 +21,7 @@ Developer Guide
    documentation
    api-design
    security
+   performance
    dependencies
    debugging
    coding-style
diff --git a/doc/sphinx-guides/source/developers/performance.rst b/doc/sphinx-guides/source/developers/performance.rst
new file mode 100644
index 00000000000..aa50cd6e40c
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/performance.rst
@@ -0,0 +1,196 @@
+Performance
+===========
+
+`Performance is a feature <https://blog.codinghorror.com/performance-is-a-feature/>`_ was a mantra when Stack Overflow was being developed. We endeavor to do the same with Dataverse!
+
+In this section we collect ideas and share practices for improving performance.
+
+.. contents:: |toctitle|
+        :local:
+
+Problem Statement
+-----------------
+
+Performance has always been important to the Dataverse Project, but results have been uneven. We've seen enough success in the marketplace that performance must be adequate, but internally we sometimes refer to Dataverse as a pig. �
+
+Current Practices
+-----------------
+
+We've adopted a number of practices to help us maintain our current level of performance and most should absolutely continue in some form, but challenges mentioned throughout should be addressed to further improve performance.
+
+Cache When You Can
+~~~~~~~~~~~~~~~~~~
+
+The Metrics API, for example, caches values for 7 days by default. We took a look at JSR 107 (JCache - Java Temporary Caching API) in `#2100 <https://github.com/IQSS/dataverse/issues/2100>`_. We're aware of the benefits of caching.
+
+Use Async
+~~~~~~~~~
+
+We index datasets (and all objects) asynchronously. That is, we let changes persist in the database and afterward copy the data into Solr.
+
+Use a Queue
+~~~~~~~~~~~
+
+We use a JMS queue for when ingesting tabular files. We've talked about adding a queue (even `an external queue <https://github.com/IQSS/dataverse/issues/1100%23issuecomment-311341995>`_) for indexing, DOI registration, and other services.
+
+Offload Expensive Operations Outside the App Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When operations are computationally expensive, we have realized performance gains by offloading them to systems outside of the core code. For example, rather than having files pass through our application server when they are downloaded, we use direct download so that client machines download files directly from S3. (We use the same trick with upload.) When a client downloads multiple files, rather than zipping them within the application server as before, we now have a separate "zipper" process that does this work out of band.
+
+Drop to Raw SQL as Necessary
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We aren't shy about writing raw SQL queries when necessary. We've written `querycount <https://github.com/IQSS/dataverse/blob/v6.0/scripts/database/querycount/README.txt>`_  scripts to help identify problematic queries and mention slow query log at :doc:`/admin/monitoring`.
+
+Add Indexes to Database Tables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There was a concerted effort in `#1880 <https://github.com/IQSS/dataverse/issues/1880>`_ to add indexes to a large number of columns, but it's something we're mindful of, generally. Perhaps we could use some better detection of when indexes would be valuable.
+
+Find Bottlenecks with a Profiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+VisualVM is popular and bundled with Netbeans. Many options are available including `JProfiler <https://github.com/IQSS/dataverse/pull/9413>`_.
+
+Warn Developers in Code Comments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For code that has been optimized for performance, warnings are sometimes inserted in the form of comments for future developers to prevent backsliding.
+
+Write Docs for Devs about Perf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Like this doc. :)
+
+Sometimes perf is written about in other places, such as :ref:`avoid-efficiency-issues-with-render-logic-expressions`.
+
+Horizontal Scaling of App Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We've made it possible to run more than one application server, though it requires some special configuration. This way load can be spread out across multiple servers. For details, see :ref:`multiple-app-servers` in the Installation Guide.
+
+Code Review and QA
+~~~~~~~~~~~~~~~~~~
+
+Before code is merged, while it is in review or QA, if a performance problem is detected (usually on an ad hoc basis), the code is returned to the developer for improvement. Developers and reviewers typically do not have many tools at their disposal to test code changes against anything close to production data. QA maintains a machine with a copy of production data but tests against smaller data unless a performance problem is suspected.
+
+A new QA guide is coming in https://github.com/IQSS/dataverse/pull/10103
+
+Locust Testing at Release Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As one of the final steps in preparing for a release, QA runs performance tests using a tool called Locust as explained the Developer Guide (see :ref:`locust`). The tests are not comprehensive, testing only a handful of pages with anonymous users, but they increase confidence that the upcoming release is not drastically slower than previous releases.
+
+Issue Tracking and Prioritization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Performance issues are tracked in our issue tracker under the `Feature: Performance & Stability <https://github.com/IQSS/dataverse/issues?q=is%3Aopen+is%3Aissue+label%3A%22Feature%3A+Performance+%26+Stability%22>`_ label (e.g. `#7788 <https://github.com/IQSS/dataverse/issues/7788>`_). That way, we can track performance problems throughout the application. Unfortunately, the pain is often felt by users in production before we realize there is a problem. As needed, performance issues are prioritized to be included in a sprint, to \ `speed up the collection page <https://github.com/IQSS/dataverse/pull/8143>`_, for example.
+
+Document Performance Tools
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the :doc:`/admin/monitoring` page section of the Admin Guide we describe how to set up Munin for monitoring performance of an operating system. We also explain how to set up Performance Insights to monitor AWS RDS (PostgreSQL as a service, in our case). In the :doc:`/developers/tools` section of the Developer Guide, we have documented how to use Eclipse Memory Analyzer Tool (MAT), SonarQube, jmap, and jstat.
+
+Google Analytics
+~~~~~~~~~~~~~~~~
+
+Emails go to a subset of the team monthly with subjects like "Your September Search performance for https://dataverse.harvard.edu" with a link to a report but it's mostly about the number clicks, not how fast the site is. It's unclear if it provides any value with regard to performance.
+
+Abandoned Tools and Practices
+-----------------------------
+
+New Relic
+~~~~~~~~~
+
+For many years Harvard Dataverse was hooked up to New Relic, a tool that promises all-in-one observability, according to their `website <https://newrelic.com>`_. In practice, we didn't do much with `the data <https://github.com/IQSS/dataverse/issues/3665>`_.
+
+Areas of Particular Concern
+---------------------------
+
+Command Engine Execution Rate Metering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We'd like to rate limit commands (CreateDataset, etc.) so that we can keep them at a reasonable level (`#9356 <https://github.com/IQSS/dataverse/issues/9356>`_). This is similar to how many APIs are rate limited, such as the GitHub API.
+
+Solr
+~~~~
+
+While in the past Solr performance hasn't been much of a concern, in recent years we've noticed performance problems when Harvard Dataverse is under load. We are investigating in `#9635 <https://github.com/IQSS/dataverse/issues/9635>`_.
+
+Datasets with Large Numbers of Files or Versions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We'd like to scale Dataverse to better handle large number of files or versions (`#9763 <https://github.com/IQSS/dataverse/issues/9763>`_).
+
+Withstanding Bots
+~~~~~~~~~~~~~~~~~
+
+Google bot, etc.
+
+Suggested Practices
+-------------------
+
+Many of our current practices should remain in place unaltered. Others could use some refinement. Some new practices should be adopted as well. Here are some suggestions.
+
+Implement the Frontend Plan for Performance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `Dataverse - SPA MVP Definition doc <https://docs.google.com/document/d/1WnJzLeVK5eVP4_10eX6BwPAnmiamO1n2uGzcwrAsucQ/edit?usp%3Dsharing>`_  has some ideas around how to achieve good performance for the new front end in the areas of rendering, monitoring,file upload/download, pagination, and caching. We should create as many issues as necessary in the frontend repo and work on them in time. The doc recommends the use of `React Profiler <https://legacy.reactjs.org/blog/2018/09/10/introducing-the-react-profiler.html>`_ and other tools. Not mentioned is https://pagespeed.web.dev but we can investigate it as well. See also `#183 <https://github.com/IQSS/dataverse-frontend/issues/183>`_, a parent issue about performance. In `#184 <https://github.com/IQSS/dataverse-frontend/issues/184>`_  we plan to compare the performance of the old JSF UI vs. the new React UI. Cypress plugins for load testing could be investigated.
+
+Set up Query Counter in Jenkins
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+See countquery script above. See also https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/query_count.out
+
+Show the plot over time. Make spikes easily apparent. 320,035 queries as of this writing.
+
+Count Database Queries per API Test
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Is it possible? Just a thought.
+
+Teach Developers How to Do Performance Testing Locally
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Do developers know how to use a profiler? Should they use `JMeter <https://github.com/BU-NU-CLOUD-SP18/Dataverse-Scaling%23our-project-video>`_? `statsd-jvm-profiler <https://github.com/etsy/statsd-jvm-profiler>`_? How do you run our :ref:`locust` tests? Should we continue using that tool? Give developers time and space to try out tools and document any tips along the way. For this stage, small data is fine.
+
+Automate Performance Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We are already using two excellent continuous integration (CI) tools, Jenkins and GitHub Actions, to test our code. We should add performance testing into the mix (`#4201 <https://github.com/IQSS/dataverse/issues/4201>`_ is an old issue for this but we can open a fresh one). Currently we test every commit on every PR and we should consider if this model makes sense since performance testing will likely take longer to run than regular tests. Once developers are comfortable with their favorite tools, we can pick which ones to automate.
+
+Make Production Data or Equivalent Available to Developers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If developers are only testing small amounts of data on their laptops, it's hard to detect performance problems. Not every bug fix requires access to data similar to production, but it should be made available. This is not a trivial task! If we are to use actual production data, we need to be very careful to de-identify it. If we start with our `sample-data <https://github.com/IQSS/dataverse-sample-data>`_  repo instead, we'll need to figure out how to make sure we cover cases like many files, many versions, etc.
+
+Automate Performance Testing with Production Data or Equivalent
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Hopefully the environment developers use with production data or equivalent can be made available to our CI tools. Perhaps these tests don't need to be run on every commit to every pull request, but they should be run regularly.
+
+Use Monitoring as Performance Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Monitoring can be seen as a form of testing. How long is a round trip ping to production? What is the Time to First Byte? First Contentful Paint? Largest Contentful Paint? Time to Interactive? We now have a beta server that we could monitor continuously to know if our app is getting faster or slower over time. Should our monitoring of production servers be improved?
+
+Learn from Training and Conferences
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Most likely there is training available that is oriented toward performance. The subject of performance often comes up at conferences as well.
+
+Learn from the Community How They Monitor Performance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some members of the Dataverse community are likely users of newish tools like the ELK stack (Elasticsearch, Logstash, and Kibana), the TICK stack (Telegraph InfluxDB Chronograph and Kapacitor), GoAccess, Prometheus, Graphite, and more we haven't even heard of. In the :doc:`/admin/monitoring` section of the Admin Guide, we already encourage the community to share findings (, but we could dedicate time to this topic at our annual meeting or community calls.
+
+Teach the Community to Do Performance Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have a worldwide community of developers. We should do what we can in the form of documentation and other resources to help them develop performant code.
+
+Conclusion
+----------
+
+Given its long history, Dataverse has encountered many performance problems over the years. The core team is conversant in how to make the app more performant, but investment in learning additional tools and best practices would likely yield dividends. We should automate our performance testing, catching more problems before code is merged.
diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index abecaa09fad..94b94eb0ab6 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -426,6 +426,8 @@ target/coverage-it/index.html is the place to start reading the code coverage re
 Load/Performance Testing
 ------------------------
 
+.. _locust:
+
 Locust
 ~~~~~~
 
diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst
index e1ee40cafa5..764434d1896 100755
--- a/doc/sphinx-guides/source/developers/tips.rst
+++ b/doc/sphinx-guides/source/developers/tips.rst
@@ -238,6 +238,8 @@ with the following code in ``SettingsWrapper.java``:
 
 A more serious example would be direct calls to PermissionServiceBean methods used in render logic expressions. This is something that has happened and caused some problems in real life. A simple permission service lookup (for example, whether a user is authorized to create a dataset in the current dataverse) can easily take 15 database queries. Repeated multiple times, this can quickly become a measurable delay in rendering the page. PermissionsWrapper must be used exclusively for any such lookups from JSF pages.
 
+See also :doc:`performance`.
+
 ----
 
 Previous: :doc:`dev-environment` | Next: :doc:`troubleshooting`
diff --git a/doc/sphinx-guides/source/installation/advanced.rst b/doc/sphinx-guides/source/installation/advanced.rst
index 87f2a4fd0ab..3de5d0ea07c 100644
--- a/doc/sphinx-guides/source/installation/advanced.rst
+++ b/doc/sphinx-guides/source/installation/advanced.rst
@@ -7,6 +7,8 @@ Advanced installations are not officially supported but here we are at least doc
 .. contents:: |toctitle|
 	:local:
 
+.. _multiple-app-servers:
+
 Multiple App Servers
 --------------------
 

From 3e004c2c1aa7bdb720298dbd00d98d8ce48f1428 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 20 Nov 2023 17:23:09 -0500
Subject: [PATCH 762/922] set/delete apis for quotas (#8549)

---
 .../harvard/iq/dataverse/api/Dataverses.java  | 41 +++++++++++++++++++
 .../iq/dataverse/storageuse/StorageUse.java   |  3 +-
 src/main/java/propertyFiles/Bundle.properties |  4 ++
 3 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index d0711aefa5f..6539e42b07d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -41,9 +41,11 @@
 import edu.harvard.iq.dataverse.engine.command.impl.CreateExplicitGroupCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.DeleteCollectionQuotaCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseLinkingDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteExplicitGroupCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionQuotaCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetRootCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseStorageSizeCommand;
@@ -63,6 +65,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RemoveRoleAssigneesFromExplicitGroupCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.SetCollectionQuotaCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseMetadataBlocksCommand;
@@ -937,6 +940,44 @@ public Response getStorageSize(@Context ContainerRequestContext crc, @PathParam(
                 execCommand(new GetDataverseStorageSizeCommand(req, findDataverseOrDie(dvIdtf), includeCached)))), getRequestUser(crc));
     }
     
+    @GET
+    @AuthRequired
+    @Path("{identifier}/storage/quota")
+    public Response getCollectionQuota(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) throws WrappedResponse {
+        try {
+            Long bytesAllocated = execCommand(new GetCollectionQuotaCommand(createDataverseRequest(getRequestUser(crc)), findDataverseOrDie(dvIdtf)));
+            if (bytesAllocated != null) {
+                return ok(MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.storage.quota.allocation"),bytesAllocated));
+            }
+            return ok(BundleUtil.getStringFromBundle("dataverse.storage.quota.notdefined"));
+        } catch (WrappedResponse ex) {
+            return ex.getResponse();
+        }
+    }
+    
+    @POST
+    @AuthRequired
+    @Path("{identifier}/storage/quota/{bytesAllocated}")
+    public Response setCollectionQuota(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, @PathParam("bytesAllocated") Long bytesAllocated) throws WrappedResponse {
+        try {
+            execCommand(new SetCollectionQuotaCommand(createDataverseRequest(getRequestUser(crc)), findDataverseOrDie(dvIdtf), bytesAllocated));
+            return ok(BundleUtil.getStringFromBundle("dataverse.storage.quota.updated"));
+        } catch (WrappedResponse ex) {
+            return ex.getResponse();
+        }
+    }
+    
+    @DELETE
+    @AuthRequired
+    @Path("{identifier}/storage/quota")
+    public Response deleteCollectionQuota(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) throws WrappedResponse {
+        try {
+            execCommand(new DeleteCollectionQuotaCommand(createDataverseRequest(getRequestUser(crc)), findDataverseOrDie(dvIdtf)));
+            return ok(BundleUtil.getStringFromBundle("dataverse.storage.quota.deleted"));
+        } catch (WrappedResponse ex) {
+            return ex.getResponse();
+        }
+    }
     
     @GET
     @AuthRequired
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
index 2633e3e026b..c91ff69b0ae 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
@@ -23,7 +23,8 @@
  */
 @NamedQueries({
     @NamedQuery(name = "StorageUse.findByteSizeByDvContainerId",query = "SELECT su.sizeInBytes FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId "),
-    @NamedQuery(name = "StorageUse.findByDvContainerId",query = "SELECT su FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId ")
+    @NamedQuery(name = "StorageUse.findByDvContainerId",query = "SELECT su FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId "),
+    @NamedQuery(name = "StorageUse.incrementByteSizeByDvContainerId", query = "UPDATE StorageUse SET sizeInBytes = (sizeInBytes + :fileSize) WHERE dvObjectContainer.id=dv.ObjectId")
 })
 @Entity
 public class StorageUse implements Serializable {
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 79887f7e76c..c5b94b7be58 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -921,6 +921,10 @@ dataverse.update.failure=This dataverse was not able to be updated.
 dataverse.selected=Selected
 dataverse.listing.error=Fatal error trying to list the contents of the dataverse. Please report this error to the Dataverse administrator.
 dataverse.datasize=Total size of the files stored in this dataverse: {0} bytes
+dataverse.storage.quota.allocation=Total quota allocation for this collection: {0} bytes
+dataverse.storage.quota.notdefined=No quota defined for this collection
+dataverse.storage.quota.updated=Storage quota successfully updated for the collection
+dataverse.storage.quota.deleted=Storage quota successfully disabled the collection
 dataverse.datasize.ioerror=Fatal IO error while trying to determine the total size of the files stored in the dataverse. Please report this error to the Dataverse administrator.
 dataverse.inherited=(inherited from enclosing Dataverse)
 dataverse.default=(Default)

From fb952a20720505e432196a77254b19947cc623a6 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 20 Nov 2023 17:25:07 -0500
Subject: [PATCH 763/922] more commands (#8549)

---
 .../impl/DeleteCollectionQuotaCommand.java    | 49 +++++++++++++++
 .../impl/SetCollectionQuotaCommand.java       | 62 +++++++++++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
new file mode 100644
index 00000000000..5fcbad929a9
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
@@ -0,0 +1,49 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author landreev
+ */
+@RequiredPermissions(Permission.ManageDataversePermissions)
+public class DeleteCollectionQuotaCommand  extends AbstractVoidCommand {
+
+    private static final Logger logger = Logger.getLogger(DeleteCollectionQuotaCommand.class.getCanonicalName());
+    
+    private final Dataverse targetDataverse;
+    
+    public DeleteCollectionQuotaCommand(DataverseRequest aRequest, Dataverse target) {
+        super(aRequest, target);
+        targetDataverse = target;
+    } 
+        
+    @Override
+    public void executeImpl(CommandContext ctxt) throws CommandException {
+        if (targetDataverse == null) {
+            throw new IllegalCommandException("", this);
+        }
+        
+        StorageQuota storageQuota = targetDataverse.getStorageQuota();
+        
+        if (storageQuota != null && storageQuota.getAllocation() != null) {
+            storageQuota.setAllocation(null);
+            ctxt.em().merge(storageQuota);
+            ctxt.em().flush();
+        } 
+        // ... and if no quota was enabled on the collection - nothing to do = success
+    }    
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
new file mode 100644
index 00000000000..a134cbefdb9
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
@@ -0,0 +1,62 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author landreev
+ */
+@RequiredPermissions(Permission.ManageDataversePermissions)
+public class SetCollectionQuotaCommand  extends AbstractVoidCommand {
+
+    private static final Logger logger = Logger.getLogger(GetCollectionQuotaCommand.class.getCanonicalName());
+    
+    private final Dataverse dataverse;
+    private final Long allocation; 
+    
+    public SetCollectionQuotaCommand(DataverseRequest aRequest, Dataverse target, Long allocation) {
+        super(aRequest, target);
+        dataverse = target;
+        this.allocation = allocation; 
+    } 
+        
+    @Override
+    public void executeImpl(CommandContext ctxt) throws CommandException {
+        
+        if (dataverse == null) {
+            throw new IllegalCommandException("", this);
+        }
+        
+        if (allocation == null) {
+            throw new IllegalCommandException("", this);
+        }
+        
+        StorageQuota storageQuota = dataverse.getStorageQuota();
+        
+        if (storageQuota != null) {
+            storageQuota.setAllocation(allocation);
+            ctxt.em().merge(storageQuota);
+        } else {
+            storageQuota = new StorageQuota(); 
+            storageQuota.setDefinitionPoint(dataverse);
+            storageQuota.setAllocation(allocation);
+            dataverse.setStorageQuota(storageQuota);
+            ctxt.em().persist(storageQuota);
+        }
+        ctxt.em().flush();
+    }    
+}

From 233da54f29b8a741d65fcb54e2f394a98485d6a5 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 21 Nov 2023 10:51:10 -0500
Subject: [PATCH 764/922] add to API changelog #10104

---
 doc/sphinx-guides/source/api/changelog.rst  | 11 +++++++++--
 doc/sphinx-guides/source/api/native-api.rst |  2 ++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index d6742252d27..1726736e75c 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -5,9 +5,16 @@ API Changelog
     :local:
     :depth: 1
 
+6.1
+---
+
+Changes
+~~~~~~~
+- **/api/datasets/{id}/versions/{versionId}/citation**: This endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation. See :ref:`get-citation`.
+
 6.0
------
+---
 
 Changes
 ~~~~~~~
-- **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch.
+- **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch. See :doc:`dataaccess`.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 2e3a0b2af08..2713580f238 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -2491,6 +2491,8 @@ Get Dataset By Private URL Token
 
   curl "$SERVER_URL/api/datasets/privateUrlDatasetVersion/$PRIVATE_URL_TOKEN"
 
+.. _get-citation:
+
 Get Citation
 ~~~~~~~~~~~~
 

From d720aeaaffce7a9196769789dc888f6ea05c193a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 21 Nov 2023 11:57:14 -0500
Subject: [PATCH 765/922] rename flyway script #10093

---
 ...id_update.sql => V6.0.0.4__10093-privateurluser_id_update.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V6.0.0.3__10093-privateurluser_id_update.sql => V6.0.0.4__10093-privateurluser_id_update.sql} (100%)

diff --git a/src/main/resources/db/migration/V6.0.0.3__10093-privateurluser_id_update.sql b/src/main/resources/db/migration/V6.0.0.4__10093-privateurluser_id_update.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.0.0.3__10093-privateurluser_id_update.sql
rename to src/main/resources/db/migration/V6.0.0.4__10093-privateurluser_id_update.sql

From c82faf967d62dc7e076d59885c77548cc90e206b Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 21 Nov 2023 13:46:36 -0500
Subject: [PATCH 766/922] #9464 fix formatting

---
 .../dataverse/metrics/MetricsServiceBean.java | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index 832dda5ced9..ff283bcda80 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -207,17 +207,17 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
         // A published local datasets may have more than one released version!
         // So that's why we have to jump through some extra hoops below
         // in order to select the latest one:
-        String originClause = "(datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in\n" +
-                "(\n" +
-                "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n" +
-                "       from datasetversion\n" +
-                "       join dataset on dataset.id = datasetversion.dataset_id\n" +
-                "       join dvobject on dataset.id = dvobject.id\n" +
-                "       where versionstate='RELEASED'\n" +
-                "       	     and dvobject.harvestingclient_id is null\n" +
-                "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n" +
-                "       group by dataset_id\n" +
-                "))\n";
+        String originClause = "(datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in\n"
+                + "(\n"
+                + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
+                + "       from datasetversion\n"
+                + "       join dataset on dataset.id = datasetversion.dataset_id\n"
+                + "       join dvobject on dataset.id = dvobject.id\n"
+                + "       where versionstate='RELEASED'\n"
+                + "       	     and dvobject.harvestingclient_id is null\n"
+                + "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
+                + "       group by dataset_id\n"
+                + "))\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
             //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated

From 44a07a31b8baf84fd6bfd5fdb846d0976a52f1aa Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 21 Nov 2023 13:54:39 -0500
Subject: [PATCH 767/922] #9464 more code cleanup

---
 .../dataverse/metrics/MetricsServiceBean.java | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index ff283bcda80..2b62c6cd9a7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -51,7 +51,7 @@ public class MetricsServiceBean implements Serializable {
 
     /** Dataverses */
 
-    
+
     public JsonArray getDataversesTimeSeries(UriInfo uriInfo, Dataverse d) {
         Query query = em.createNativeQuery(""
                 + "select distinct to_char(date_trunc('month', dvobject.publicationdate),'YYYY-MM') as month, count(date_trunc('month', dvobject.publicationdate))\n"
@@ -64,7 +64,7 @@ public JsonArray getDataversesTimeSeries(UriInfo uriInfo, Dataverse d) {
         List<Object[]> results = query.getResultList();
         return MetricsUtil.timeSeriesToJson(results);
     }
-    
+
     /**
      * @param yyyymm Month in YYYY-MM format.
      * @param d
@@ -129,9 +129,9 @@ public List<Object[]> dataversesBySubject(Dataverse d) {
 
     /** Datasets */
 
-    
+
     public JsonArray getDatasetsTimeSeries(UriInfo uriInfo, String dataLocation, Dataverse d) {
-        Query query = em.createNativeQuery(                
+        Query query = em.createNativeQuery(
                 "select distinct date, count(dataset_id)\n"
                 + "from (\n"
                 + "select min(to_char(COALESCE(releasetime, createtime), 'YYYY-MM')) as date, dataset_id\n"
@@ -149,8 +149,8 @@ public JsonArray getDatasetsTimeSeries(UriInfo uriInfo, String dataLocation, Dat
         List<Object[]> results = query.getResultList();
         return MetricsUtil.timeSeriesToJson(results);
     }
-    
-    
+
+
     /**
      * @param yyyymm Month in YYYY-MM format.
      * @param d
@@ -180,10 +180,10 @@ public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
         // But do not use this notation if you need the values returned to
         // meaningfully identify the datasets!
 
-  
+
         Query query = em.createNativeQuery(
-                        
-                        
+
+
                 "select count(*)\n"
                         + "from (\n"
                         + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
@@ -214,7 +214,7 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
                 + "       join dataset on dataset.id = datasetversion.dataset_id\n"
                 + "       join dvobject on dataset.id = dvobject.id\n"
                 + "       where versionstate='RELEASED'\n"
-                + "       	     and dvobject.harvestingclient_id is null\n"
+                + "       	     and dvobject.harvestingclient_id is null"
                 + "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
                 + "       group by dataset_id\n"
                 + "))\n";
@@ -313,7 +313,7 @@ public JsonArray filesTimeSeries(Dataverse d) {
         return MetricsUtil.timeSeriesToJson(results);
     }
 
-    
+
     /**
      * @param yyyymm Month in YYYY-MM format.
      * @param d
@@ -390,7 +390,7 @@ public JsonArray filesByType(Dataverse d) {
         return jab.build();
 
     }
-    
+
     public JsonArray filesByTypeTimeSeries(Dataverse d, boolean published) {
         Query query = em.createNativeQuery("SELECT DISTINCT to_char(" + (published ? "ob.publicationdate" : "ob.createdate") + ",'YYYY-MM') as date, df.contenttype, count(df.id), coalesce(sum(df.filesize),0) "
                 + " FROM DataFile df, DvObject ob"
@@ -403,13 +403,13 @@ public JsonArray filesByTypeTimeSeries(Dataverse d, boolean published) {
         logger.log(Level.FINE, "Metric query: {0}", query);
         List<Object[]> results = query.getResultList();
         return MetricsUtil.timeSeriesByTypeToJson(results);
-        
+
     }
-    /** Downloads 
+    /** Downloads
      * @param d
      * @throws ParseException */
 
-    
+
     public JsonArray downloadsTimeSeries(Dataverse d) {
         // ToDo - published only?
         Query earlyDateQuery = em.createNativeQuery(""
@@ -433,11 +433,11 @@ public JsonArray downloadsTimeSeries(Dataverse d) {
         List<Object[]> results = query.getResultList();
         return MetricsUtil.timeSeriesToJson(results);
     }
-    
+
     /*
      * This includes getting historic download without a timestamp if query
      * is earlier than earliest timestamped record
-     * 
+     *
      * @param yyyymm Month in YYYY-MM format.
      */
     public long downloadsToMonth(String yyyymm, Dataverse d) throws ParseException {
@@ -460,7 +460,7 @@ public long downloadsToMonth(String yyyymm, Dataverse d) throws ParseException {
                         + "where (date_trunc('month', responsetime) <=  to_date('" + yyyymm + "','YYYY-MM')"
                         + "or responsetime is NULL)\n" // includes historic guestbook records without date
                         + "and eventtype!='" + GuestbookResponse.ACCESS_REQUEST +"'\n"
-                    + ((d==null) ? ";": "AND dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ");") 
+                    + ((d==null) ? ";": "AND dataset_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ");")
                 );
                 logger.log(Level.FINE, "Metric query: {0}", query);
                 return (long) query.getSingleResult();
@@ -488,7 +488,7 @@ public long downloadsPastDays(int days, Dataverse d) {
 
         return (long) query.getSingleResult();
     }
-    
+
     public JsonArray fileDownloadsTimeSeries(Dataverse d, boolean uniqueCounts) {
         Query query = em.createNativeQuery("select distinct to_char(gb.responsetime, 'YYYY-MM') as date, ob.id, ob.protocol || ':' || ob.authority || '/' || ob.identifier as pid, count(" + (uniqueCounts ? "distinct email" : "*") + ") "
                 + " FROM guestbookresponse gb, DvObject ob"
@@ -502,7 +502,7 @@ public JsonArray fileDownloadsTimeSeries(Dataverse d, boolean uniqueCounts) {
         return MetricsUtil.timeSeriesByIDAndPIDToJson(results);
 
     }
-    
+
     public JsonArray fileDownloads(String yyyymm, Dataverse d, boolean uniqueCounts) {
         Query query = em.createNativeQuery("select ob.id, ob.protocol || ':' || ob.authority || '/' || ob.identifier as pid, count(" + (uniqueCounts ? "distinct email" : "*") + ") "
                 + " FROM guestbookresponse gb, DvObject ob"
@@ -544,7 +544,7 @@ public JsonArray uniqueDownloadsTimeSeries(Dataverse d) {
         return MetricsUtil.timeSeriesByPIDToJson(results);
 
     }
-    
+
     public JsonArray uniqueDatasetDownloads(String yyyymm, Dataverse d) {
 
     //select distinct count(distinct email),dataset_id, date_trunc('month', responsetime)  from guestbookresponse group by dataset_id, date_trunc('month',responsetime) order by dataset_id,date_trunc('month',responsetime);
@@ -572,10 +572,10 @@ public JsonArray uniqueDatasetDownloads(String yyyymm, Dataverse d) {
         return jab.build();
 
     }
-    
-    //MDC 
-    
-    
+
+    //MDC
+
+
     public JsonArray mdcMetricTimeSeries(MetricType metricType, String country, Dataverse d) {
         Query query = em.createNativeQuery("SELECT distinct substring(monthyear from 1 for 7) as date, coalesce(sum(" + metricType.toString() + "),0) as count FROM DatasetMetrics\n"
                 + ((d == null) ? "" : "WHERE dataset_id in ( " + getCommaSeparatedIdStringForSubtree(d, "Dataset") + ")\n")
@@ -747,7 +747,7 @@ public Metric getMetric(String name, String dataLocation, String dayString, Data
     // https://github.com/DANS-KNAW/dataverse/blob/dans-develop/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsDansServiceBean.java
 
     /**
-     * 
+     *
      * @param dvId - parent dataverse id
      * @param dtype - type of object to return 'Dataverse' or 'Dataset'
      * @return - list of objects of specified type included in the subtree (includes parent dataverse if dtype is 'Dataverse')
@@ -769,7 +769,7 @@ private String getCommaSeparatedIdStringForSubtree(Dataverse d, String dtype) {
     }
 
     private List<Integer> getChildrenIdsRecursively(Long dvId, String dtype, DatasetVersion.VersionState versionState) {
-        
+
         //Intended to be called only with dvId != null
         String sql = "WITH RECURSIVE querytree AS (\n"
                 + "     SELECT id, dtype, owner_id, publicationdate\n"

From 7d687e9e06f9a70724650b13f91eab5a2a767e97 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 21 Nov 2023 13:59:38 -0500
Subject: [PATCH 768/922] #9464 third time's the charm?

---
 .../edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index 2b62c6cd9a7..a50797fe443 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -214,8 +214,8 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
                 + "       join dataset on dataset.id = datasetversion.dataset_id\n"
                 + "       join dvobject on dataset.id = dvobject.id\n"
                 + "       where versionstate='RELEASED'\n"
-                + "       	     and dvobject.harvestingclient_id is null"
-                + "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
+                + "       and dvobject.harvestingclient_id is null"
+                + "       and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
                 + "       group by dataset_id\n"
                 + "))\n";
 

From 1c2a20d940632da96c9d56c1bb40367c7902b398 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 21 Nov 2023 14:25:26 -0500
Subject: [PATCH 769/922] fix formatting (indentation) #10093

---
 .../iq/dataverse/api/auth/SignedUrlAuthMechanism.java       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java
index e2c2f2381d8..258661f6495 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/auth/SignedUrlAuthMechanism.java
@@ -61,9 +61,9 @@ private User getAuthenticatedUserFromSignedUrl(ContainerRequestContext container
         String userId = uriInfo.getQueryParameters().getFirst(SIGNED_URL_USER);
         User targetUser = null; 
         ApiToken userApiToken = null;
-        if(!userId.startsWith(PrivateUrlUser.PREFIX)) {
-        targetUser = authSvc.getAuthenticatedUser(userId);
-        userApiToken = authSvc.findApiTokenByUser((AuthenticatedUser)targetUser);
+        if (!userId.startsWith(PrivateUrlUser.PREFIX)) {
+            targetUser = authSvc.getAuthenticatedUser(userId);
+            userApiToken = authSvc.findApiTokenByUser((AuthenticatedUser) targetUser);
         } else {
             PrivateUrl privateUrl = privateUrlSvc.getPrivateUrlFromDatasetId(Long.parseLong(userId.substring(PrivateUrlUser.PREFIX.length())));
             userApiToken = new ApiToken();

From 949b0885f84d953ab51cc458b4f61c5d376b12a9 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 21 Nov 2023 14:25:59 -0500
Subject: [PATCH 770/922] give Signed URLs a "ref" in the guides, link to it
 #10093

---
 doc/sphinx-guides/source/api/auth.rst         |  5 ++++
 .../source/api/external-tools.rst             | 26 ++++++++++++-------
 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst
index eced7afbbcf..eae3bd3c969 100644
--- a/doc/sphinx-guides/source/api/auth.rst
+++ b/doc/sphinx-guides/source/api/auth.rst
@@ -80,3 +80,8 @@ To test if bearer tokens are working, you can try something like the following (
   export TOKEN=`curl -s -X POST --location "http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token" -H "Content-Type: application/x-www-form-urlencoded" -d "username=user&password=user&grant_type=password&client_id=test&client_secret=94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8" | jq '.access_token' -r | tr -d "\n"`
   
   curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/api/users/:me
+
+Signed URLs
+-----------
+
+See :ref:`signed-urls`.
diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst
index d802bc8714a..ae0e44b36aa 100644
--- a/doc/sphinx-guides/source/api/external-tools.rst
+++ b/doc/sphinx-guides/source/api/external-tools.rst
@@ -160,17 +160,25 @@ Authorization Options
 
 When called for datasets or data files that are not public (i.e. in a draft dataset or for a restricted file), external tools are allowed access via the user's credentials. This is accomplished by one of two mechanisms:
 
-* Signed URLs (more secure, recommended)
+.. _signed-urls:
 
-  - Configured via the ``allowedApiCalls`` section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified.
-  - For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest.
-  - For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
+Signed URLs
+^^^^^^^^^^^
 
-* API Token (deprecated, less secure, not recommended)
+The signed URL mechanism is more secure than exposing API tokens and therefore recommended.
 
-  - Configured via the ``queryParameters`` by including an ``{apiToken}`` value. When this is present Dataverse will send the user's apiToken to the tool. With the user's API token, the tool can perform any action via the Dataverse API that the user could. External tools configured via this method should be assessed for their trustworthiness.
-  - For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
-  - For tools invoked via POST, Dataverse will send a JSON body including the apiToken.
+- Configured via the ``allowedApiCalls`` section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified.
+- For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest.
+- For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
+
+API Token
+^^^^^^^^^
+
+The API token mechanism is deprecated. Because it is less secure than signed URLs, it is not recommended for new external tools.
+
+- Configured via the ``queryParameters`` by including an ``{apiToken}`` value. When this is present Dataverse will send the user's apiToken to the tool. With the user's API token, the tool can perform any action via the Dataverse API that the user could. External tools configured via this method should be assessed for their trustworthiness.
+- For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
+- For tools invoked via POST, Dataverse will send a JSON body including the apiToken.
 
 Internationalization of Your External Tool
 ++++++++++++++++++++++++++++++++++++++++++
@@ -187,7 +195,7 @@ Using Example Manifests to Get Started
 ++++++++++++++++++++++++++++++++++++++
 
 Again, you can use :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` or :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as a starting point for your own manifest file.
-Additional working examples, including ones using signedUrls, are available at https://github.com/gdcc/dataverse-previewers .
+Additional working examples, including ones using :ref:`signed-urls`, are available at https://github.com/gdcc/dataverse-previewers .
 
 Testing Your External Tool
 --------------------------

From 551194671f69b2bc9a400a1dc5e52926f0867c8e Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 21 Nov 2023 14:26:44 -0500
Subject: [PATCH 771/922] tweak release note, add issue numbers #10093 #10045

---
 doc/release-notes/10093-signedUrl_improvements.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/release-notes/10093-signedUrl_improvements.md b/doc/release-notes/10093-signedUrl_improvements.md
index 8f6ae89f981..26a17c65e3f 100644
--- a/doc/release-notes/10093-signedUrl_improvements.md
+++ b/doc/release-notes/10093-signedUrl_improvements.md
@@ -1,5 +1,5 @@
 A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
 
-SignedUrls can now be used with PrivateUrl access, i.e. allowing PrivateUrl users to view Previews when they are configured to use signedUrls
+SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
 
-Launching a Dataset-level Configuration tool will automatically generate an api key when needed. This is consistent with how other types of tools work.
+Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.

From 31e1a4fede34960671f886e1c4fdfee51a7ffc25 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Tue, 21 Nov 2023 14:26:58 -0500
Subject: [PATCH 772/922] Add validation for entered coordinate values in
 Geospatial metadata

---
 .../dataverse/DatasetFieldValueValidator.java | 84 ++++++++++++++++++-
 .../iq/dataverse/search/IndexServiceBean.java | 34 ++------
 src/main/java/propertyFiles/Bundle.properties |  1 +
 .../DatasetFieldValueValidatorTest.java       | 11 +++
 .../search/IndexServiceBeanTest.java          | 50 ++++++++---
 5 files changed, 138 insertions(+), 42 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
index b6c21014f04..7941fa5efa1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
@@ -8,9 +8,7 @@
 import edu.harvard.iq.dataverse.DatasetFieldType.FieldType;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.GregorianCalendar;
+import java.util.*;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import jakarta.validation.ConstraintValidator;
@@ -34,7 +32,6 @@ public void initialize(ValidateDatasetFieldType constraintAnnotation) {
     }
 
     public boolean isValid(DatasetFieldValue value, ConstraintValidatorContext context) {
-
         context.disableDefaultConstraintViolation(); // we do this so we can have different messages depending on the different issue
 
         boolean lengthOnly = false;
@@ -55,6 +52,33 @@ public boolean isValid(DatasetFieldValue value, ConstraintValidatorContext conte
             return true;
         }
 
+        // verify no junk in individual fields and values are within range
+        if (dsfType.getName() != null && (dsfType.getName().equals(DatasetFieldConstant.northLatitude) || dsfType.getName().equals(DatasetFieldConstant.southLatitude) ||
+                dsfType.getName().equals(DatasetFieldConstant.westLongitude) || dsfType.getName().equals(DatasetFieldConstant.eastLongitude))) {
+            try {
+                verifyBoundingBoxCoordinatesWithinRange(dsfType.getName(), value.getValue());
+            } catch (IllegalArgumentException iae) {
+                    try {
+                        context.buildConstraintViolationWithTemplate(dsfType.getDisplayName() + "  " + BundleUtil.getStringFromBundle("dataset.metadata.invalidEntry")).addConstraintViolation();
+                    } catch (NullPointerException e) {
+                    }
+                    return false;
+            }
+        }
+
+        // validate fields that are siblings and depend on each others values
+        if (value.getDatasetField().getParentDatasetFieldCompoundValue() != null) {
+            Optional<String> failureMessage = validateChildConstraints(value.getDatasetField());
+            if (failureMessage.isPresent()) {
+                try {
+                    context.buildConstraintViolationWithTemplate(dsfType.getParentDatasetFieldType().getDisplayName() +  "  " +
+                            BundleUtil.getStringFromBundle(failureMessage.get()) ).addConstraintViolation();
+                } catch (NullPointerException npe) {
+                }
+                return false;
+            }
+        }
+
         if (fieldType.equals(FieldType.TEXT) && !lengthOnly && value.getDatasetField().getDatasetFieldType().getValidationFormat() != null) {
             boolean valid = value.getValue().matches(value.getDatasetField().getDatasetFieldType().getValidationFormat());
             if (!valid) {
@@ -216,4 +240,56 @@ public boolean isValidAuthorIdentifier(String userInput, Pattern pattern) {
         return pattern.matcher(userInput).matches();
     }
 
+    // Validate child fields against each other and return failure message or null if success
+    public Optional<String> validateChildConstraints(DatasetField dsf) {
+        final String fieldName = dsf.getDatasetFieldType().getName() != null ? dsf.getDatasetFieldType().getName() : "";
+        Optional<String> returnFailureMessage = Optional.empty();
+
+        // Validate Child Constraint for Geospatial Bounding Box
+        // validate the four points of the box to insure proper layout
+        // only validate on one value for speed. picked northLatitude since it was at the bottom of the UI
+        if (fieldName.equals(DatasetFieldConstant.northLatitude)) {
+            final String failureMessage = "dataset.metadata.invalidGeospatialCoordinates";
+            DatasetFieldCompoundValue cv = dsf.getParentDatasetFieldCompoundValue();
+            List<DatasetField> cdsf = cv.getChildDatasetFields();
+            try {
+                if (cdsf.size() == 4) {
+                    if (!validateBoundingBox(cdsf.get(0).getValue(), cdsf.get(1).getValue(), cdsf.get(2).getValue(), cdsf.get(3).getValue())) {
+                        returnFailureMessage = Optional.of(failureMessage);
+                    }
+                }
+            } catch (IllegalArgumentException e) { // IllegalArgumentException NumberFormatException
+                returnFailureMessage = Optional.of(failureMessage);
+            }
+        }
+
+        return returnFailureMessage;
+    }
+
+    public static boolean validateBoundingBox(final String westLon, final String eastLon, final String northLat, final String southLat) {
+        boolean returnVal = false;
+
+        try {
+            Float west = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.westLongitude, westLon);
+            Float east = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.eastLongitude, eastLon);
+            Float north = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.northLatitude, northLat);
+            Float south = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.southLatitude, southLat);
+            returnVal = east < west && south < north;
+        } catch (IllegalArgumentException e) {
+            returnVal = false;
+        }
+
+        return returnVal;
+    }
+
+    private static Float verifyBoundingBoxCoordinatesWithinRange(final String name, final String value) throws IllegalArgumentException {
+        int max = name.equals(DatasetFieldConstant.westLongitude) || name.equals(DatasetFieldConstant.eastLongitude) ? 180 : 90;
+        int min = max * -1;
+
+        final Float returnVal = value != null ? Float.parseFloat(value) : min; // defaults to min if value is missing
+        if (returnVal < min || returnVal > max) {
+            throw new IllegalArgumentException(String.format("Value (%s) not in range (%s-%s)", returnVal, min, max));
+        }
+        return returnVal;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index d6d0be7a17b..8138a6f414f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -1,27 +1,6 @@
 package edu.harvard.iq.dataverse.search;
 
-import edu.harvard.iq.dataverse.ControlledVocabularyValue;
-import edu.harvard.iq.dataverse.DataFile;
-import edu.harvard.iq.dataverse.DataFileServiceBean;
-import edu.harvard.iq.dataverse.DataFileTag;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetField;
-import edu.harvard.iq.dataverse.DatasetFieldCompoundValue;
-import edu.harvard.iq.dataverse.DatasetFieldConstant;
-import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
-import edu.harvard.iq.dataverse.DatasetFieldType;
-import edu.harvard.iq.dataverse.DatasetLinkingServiceBean;
-import edu.harvard.iq.dataverse.DatasetServiceBean;
-import edu.harvard.iq.dataverse.DatasetVersion;
-import edu.harvard.iq.dataverse.Dataverse;
-import edu.harvard.iq.dataverse.DataverseLinkingServiceBean;
-import edu.harvard.iq.dataverse.DataverseServiceBean;
-import edu.harvard.iq.dataverse.DvObject;
-import edu.harvard.iq.dataverse.DvObjectServiceBean;
-import edu.harvard.iq.dataverse.Embargo;
-import edu.harvard.iq.dataverse.FileMetadata;
-import edu.harvard.iq.dataverse.GlobalId;
-import edu.harvard.iq.dataverse.PermissionServiceBean;
+import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
@@ -35,6 +14,7 @@
 import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -1072,13 +1052,17 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set<Long
                             if(maxNorthLat==null || Float.parseFloat(maxNorthLat) < Float.parseFloat(northLat)) {
                                 maxNorthLat=northLat;
                             }
-                            //W, E, N, S
-                            solrInputDocument.addField(SearchFields.GEOLOCATION, "ENVELOPE(" + westLon + "," + eastLon + "," + northLat + "," + southLat + ")");
+
+                            if (DatasetFieldValueValidator.validateBoundingBox(westLon, eastLon, northLat, southLat)) {
+                                //W, E, N, S
+                                solrInputDocument.addField(SearchFields.GEOLOCATION, "ENVELOPE(" + westLon + "," + eastLon + "," + northLat + "," + southLat + ")");
+                            }
                         }
                     }
                     //Only one bbox per dataset
                     //W, E, N, S
-                    if ((minWestLon != null || maxEastLon != null) && (maxNorthLat != null || minSouthLat != null)) {
+                    if (DatasetFieldValueValidator.validateBoundingBox(minWestLon, maxEastLon, maxNorthLat, minSouthLat) &&
+                            (minWestLon != null || maxEastLon != null) && (maxNorthLat != null || minSouthLat != null)) {
                         solrInputDocument.addField(SearchFields.BOUNDING_BOX, "ENVELOPE(" + minWestLon + "," + maxEastLon + "," + maxNorthLat + "," + minSouthLat + ")");
                     }
 
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 79887f7e76c..15b0ebb8020 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1623,6 +1623,7 @@ dataset.metadata.alternativePersistentId.tip=A previously used persistent identi
 dataset.metadata.invalidEntry=is not a valid entry.
 dataset.metadata.invalidDate=is not a valid date. "yyyy" is a supported format.
 dataset.metadata.invalidNumber=is not a valid number.
+dataset.metadata.invalidGeospatialCoordinates=has invalid coordinates. West must be greater than East and North must be greater than South.
 dataset.metadata.invalidInteger=is not a valid integer.
 dataset.metadata.invalidURL=is not a valid URL.
 dataset.metadata.invalidEmail=is not a valid email address.
diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
index f4af88818a5..2729e5b042a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
@@ -213,4 +213,15 @@ public void testInvalidEmail() {
             assertTrue(c.getMessage().contains("email"));
         });
     }
+    @Test
+    public void testBoundingBoxValidity() {
+        // valid tests
+        assertTrue(DatasetFieldValueValidator.validateBoundingBox("180", "-180", "90", "-90"));
+        assertTrue(DatasetFieldValueValidator.validateBoundingBox("180", "-180", "90", null));
+
+        // invalid tests
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("180", "-180", "90", "junk"));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("40", "45", "90", "0"));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("360", "0", "90", "-90"));
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
index ce6005a3d11..a0330e1fcad 100644
--- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
@@ -1,17 +1,7 @@
 package edu.harvard.iq.dataverse.search;
 
-import edu.harvard.iq.dataverse.ControlledVocabularyValue;
-import edu.harvard.iq.dataverse.DOIServiceBean;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetField;
-import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
-import edu.harvard.iq.dataverse.DatasetFieldType;
-import edu.harvard.iq.dataverse.DatasetVersion;
-import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.*;
 import edu.harvard.iq.dataverse.Dataverse.DataverseType;
-import edu.harvard.iq.dataverse.DataverseServiceBean;
-import edu.harvard.iq.dataverse.GlobalId;
-import edu.harvard.iq.dataverse.MetadataBlock;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.mocks.MocksFactory;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
@@ -21,6 +11,7 @@
 import edu.harvard.iq.dataverse.util.testing.LocalJvmSettings;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.common.SolrInputDocument;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -30,8 +21,7 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Set;
+import java.util.*;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
@@ -108,6 +98,40 @@ public void TestIndexing() throws SolrServerException, IOException {
         assertTrue(indexedFields.contains("language"));
     }
 
+    @Test
+    public void testValidateBoundingBox() throws SolrServerException, IOException {
+        final IndexableDataset indexableDataset = createIndexableDataset();
+        final DatasetVersion datasetVersion = indexableDataset.getDatasetVersion();
+        DatasetField dsf = new DatasetField();
+        DatasetFieldType dsft = new DatasetFieldType(DatasetFieldConstant.geographicBoundingBox, DatasetFieldType.FieldType.TEXT, true);
+        dsf.setDatasetFieldType(dsft);
+
+        List<DatasetFieldCompoundValue> vals = new LinkedList<>();
+        DatasetFieldCompoundValue val = new DatasetFieldCompoundValue();
+        val.setParentDatasetField(dsf);
+        val.setChildDatasetFields(Arrays.asList(
+                constructBoundingBoxValue(DatasetFieldConstant.westLongitude, "34.8"),
+                constructBoundingBoxValue(DatasetFieldConstant.eastLongitude, "34.9"), // bad value. must be less than west
+                constructBoundingBoxValue(DatasetFieldConstant.northLatitude, "34.2"),
+                constructBoundingBoxValue(DatasetFieldConstant.southLatitude, "34.1")
+        ));
+        vals.add(val);
+        dsf.setDatasetFieldCompoundValues(vals);
+        datasetVersion.getDatasetFields().add(dsf);
+
+        final SolrInputDocuments docs = indexService.toSolrDocs(indexableDataset, null);
+        Optional<SolrInputDocument> doc = docs.getDocuments().stream().findFirst();
+        assertTrue(doc.isPresent());
+        assertTrue(!doc.get().containsKey("geolocation"));
+        assertTrue(!doc.get().containsKey("boundingBox"));
+    }
+    private DatasetField constructBoundingBoxValue(String datasetFieldTypeName, String value) {
+        DatasetField retVal = new DatasetField();
+        retVal.setDatasetFieldType(new DatasetFieldType(datasetFieldTypeName, DatasetFieldType.FieldType.TEXT, false));
+        retVal.setDatasetFieldValues(Collections.singletonList(new DatasetFieldValue(retVal, value)));
+        return retVal;
+    }
+
     private IndexableDataset createIndexableDataset() {
         final Dataset dataset = MocksFactory.makeDataset();
         dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL,"10.666", "FAKE/fake", "/", DOIServiceBean.DOI_RESOLVER_URL, null));

From 48ae3332964b3d218127318f92ae997f52f3d9e0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:18:32 -0500
Subject: [PATCH 773/922] add download methods + cache, tweak upload, reformat

---
 .../dataverse/globus/GlobusServiceBean.java   | 495 ++++++++++--------
 1 file changed, 264 insertions(+), 231 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 3dee3bd498f..28173aa0074 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -120,23 +120,23 @@ public void setUserTransferToken(String userTransferToken) {
         this.userTransferToken = userTransferToken;
     }
 
-    private String getRuleId(GlobusEndpoint endpoint, String principal, String permissions) throws MalformedURLException {
-       
-        String principalType="identity";
-        
+    private String getRuleId(GlobusEndpoint endpoint, String principal, String permissions)
+            throws MalformedURLException {
+
+        String principalType = "identity";
+
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access_list");
-        MakeRequestResponse result = makeRequest(url, "Bearer",
-                endpoint.getClientToken(), "GET", null);
+        MakeRequestResponse result = makeRequest(url, "Bearer", endpoint.getClientToken(), "GET", null);
         if (result.status == 200) {
             AccessList al = parseJson(result.jsonResponse, AccessList.class, false);
 
             for (int i = 0; i < al.getDATA().size(); i++) {
                 Permissions pr = al.getDATA().get(i);
-                
+
                 if ((pr.getPath().equals(endpoint.getBasePath() + "/") || pr.getPath().equals(endpoint.getBasePath()))
                         && pr.getPrincipalType().equals(principalType)
                         && ((principal == null) || (principal != null && pr.getPrincipal().equals(principal)))
-                        &&pr.getPermissions().equals(permissions)) {
+                        && pr.getPermissions().equals(permissions)) {
                     return pr.getId();
                 } else {
                     logger.fine(pr.getPath() + " === " + endpoint.getBasePath() + " == " + pr.getPrincipalType());
@@ -148,85 +148,80 @@ private String getRuleId(GlobusEndpoint endpoint, String principal, String permi
     }
 
     /*
-    public void updatePermision(AccessToken clientTokenUser, String directory, String principalType, String perm)
-            throws MalformedURLException {
-        if (directory != null && !directory.equals("")) {
-            directory = directory + "/";
-        }
-        logger.info("Start updating permissions." + " Directory is " + directory);
-        String globusEndpoint = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusEndpoint, "");
-        ArrayList<String> rules = checkPermisions(clientTokenUser, directory, globusEndpoint, principalType, null);
-        logger.info("Size of rules " + rules.size());
-        int count = 0;
-        while (count < rules.size()) {
-            logger.info("Start removing rules " + rules.get(count));
-            Permissions permissions = new Permissions();
-            permissions.setDATA_TYPE("access");
-            permissions.setPermissions(perm);
-            permissions.setPath(directory);
-
-            Gson gson = new GsonBuilder().create();
-            URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/access/"
-                    + rules.get(count));
-            logger.info("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/access/"
-                    + rules.get(count));
-            MakeRequestResponse result = makeRequest(url, "Bearer",
-                    clientTokenUser.getOtherTokens().get(0).getAccessToken(), "PUT", gson.toJson(permissions));
-            if (result.status != 200) {
-                logger.warning("Cannot update access rule " + rules.get(count));
-            } else {
-                logger.info("Access rule " + rules.get(count) + " was updated");
-            }
-            count++;
-        }
-    }
-*/
-    
-/** Call to delete a globus rule related to the specified dataset.
- * 
- * @param ruleId - Globus rule id - assumed to be associated with the dataset's file path (should not be called with a user specified rule id w/o further checking)
- * @param datasetId - the id of the dataset associated with the rule
- * @param globusLogger - a separate logger instance, may be null
- */
-public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) {
-
-    if (ruleId.length() > 0) {
-        if (dataset != null) {
-            GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
-            if (endpoint != null) {
-                String accessToken = endpoint.getClientToken();
-                if (globusLogger != null) {
-                    globusLogger.info("Start deleting permissions.");
-                }
-                try {
-                    URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId()
-                            + "/access/" + ruleId);
-                    MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "DELETE", null);
-                    if (result.status != 200) {
-                        if (globusLogger != null) {
-                            globusLogger.warning("Cannot delete access rule " + ruleId);
+     * public void updatePermision(AccessToken clientTokenUser, String directory,
+     * String principalType, String perm) throws MalformedURLException { if
+     * (directory != null && !directory.equals("")) { directory = directory + "/"; }
+     * logger.info("Start updating permissions." + " Directory is " + directory);
+     * String globusEndpoint =
+     * settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusEndpoint, "");
+     * ArrayList<String> rules = checkPermisions(clientTokenUser, directory,
+     * globusEndpoint, principalType, null); logger.info("Size of rules " +
+     * rules.size()); int count = 0; while (count < rules.size()) {
+     * logger.info("Start removing rules " + rules.get(count)); Permissions
+     * permissions = new Permissions(); permissions.setDATA_TYPE("access");
+     * permissions.setPermissions(perm); permissions.setPath(directory);
+     * 
+     * Gson gson = new GsonBuilder().create(); URL url = new
+     * URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint
+     * + "/access/" + rules.get(count));
+     * logger.info("https://transfer.api.globusonline.org/v0.10/endpoint/" +
+     * globusEndpoint + "/access/" + rules.get(count)); MakeRequestResponse result =
+     * makeRequest(url, "Bearer",
+     * clientTokenUser.getOtherTokens().get(0).getAccessToken(), "PUT",
+     * gson.toJson(permissions)); if (result.status != 200) {
+     * logger.warning("Cannot update access rule " + rules.get(count)); } else {
+     * logger.info("Access rule " + rules.get(count) + " was updated"); } count++; }
+     * }
+     */
+
+    /**
+     * Call to delete a globus rule related to the specified dataset.
+     * 
+     * @param ruleId       - Globus rule id - assumed to be associated with the
+     *                     dataset's file path (should not be called with a user
+     *                     specified rule id w/o further checking)
+     * @param datasetId    - the id of the dataset associated with the rule
+     * @param globusLogger - a separate logger instance, may be null
+     */
+    public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) {
+
+        if (ruleId.length() > 0) {
+            if (dataset != null) {
+                GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+                if (endpoint != null) {
+                    String accessToken = endpoint.getClientToken();
+                    if (globusLogger != null) {
+                        globusLogger.info("Start deleting permissions.");
+                    }
+                    try {
+                        URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId()
+                                + "/access/" + ruleId);
+                        MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "DELETE", null);
+                        if (result.status != 200) {
+                            if (globusLogger != null) {
+                                globusLogger.warning("Cannot delete access rule " + ruleId);
+                            } else {
+                                // When removed due to a cache ejection, we don't have a globusLogger
+                                logger.warning("Cannot delete access rule " + ruleId);
+                            }
                         } else {
-                            // When removed due to a cache ejection, we don't have a globusLogger
-                            logger.warning("Cannot delete access rule " + ruleId);
-                        }
-                    } else {
-                        if (globusLogger != null) {
-                            globusLogger.info("Access rule " + ruleId + " was deleted successfully");
+                            if (globusLogger != null) {
+                                globusLogger.info("Access rule " + ruleId + " was deleted successfully");
+                            }
                         }
+                    } catch (MalformedURLException ex) {
+                        logger.log(Level.WARNING,
+                                "Failed to delete access rule " + ruleId + " on endpoint " + endpoint.getId(), ex);
                     }
-                } catch (MalformedURLException ex) {
-                    logger.log(Level.WARNING,
-                            "Failed to delete access rule " + ruleId + " on endpoint " + endpoint.getId(), ex);
                 }
             }
         }
     }
-}
 
     public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int numberOfPaths) {
 
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
-        String principalType= "identity";
+        String principalType = "identity";
 
         Permissions permissions = new Permissions();
         permissions.setDATA_TYPE("access");
@@ -235,20 +230,34 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
         permissions.setPath(endpoint.getBasePath() + "/");
         permissions.setPermissions("rw");
 
+        JsonObjectBuilder response = Json.createObjectBuilder();
+        response.add("status", requestPermission(endpoint, dataset, permissions));
+        String driverId = dataset.getEffectiveStorageDriverId();
+        JsonObjectBuilder paths = Json.createObjectBuilder();
+        for (int i = 0; i < numberOfPaths; i++) {
+            String storageIdentifier = DataAccess.getNewStorageIdentifier(driverId);
+            int lastIndex = Math.max(storageIdentifier.lastIndexOf("/"), storageIdentifier.lastIndexOf(":"));
+            paths.add(storageIdentifier, endpoint.getBasePath() + "/" + storageIdentifier.substring(lastIndex + 1));
+
+        }
+        response.add("paths", paths.build());
+        return response.build();
+    }
+
+    private int requestPermission(GlobusEndpoint endpoint, Dataset dataset, Permissions permissions) {
         Gson gson = new GsonBuilder().create();
         MakeRequestResponse result = null;
-            logger.info("Start creating the rule");
-            JsonObjectBuilder response = Json.createObjectBuilder();
+        logger.info("Start creating the rule");
 
-            try {
+        try {
             URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId() + "/access");
-            result = makeRequest(url, "Bearer", endpoint.getClientToken(), "POST",
-                    gson.toJson(permissions));
+            result = makeRequest(url, "Bearer", endpoint.getClientToken(), "POST", gson.toJson(permissions));
 
-            response.add("status", result.status);
             switch (result.status) {
+            case 404:
+                logger.severe("Endpoint " + endpoint.getId() + " was not found");
+                break;
             case 400:
-
                 logger.severe("Path " + permissions.getPath() + " is not valid");
                 break;
             case 409:
@@ -260,28 +269,17 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
                     permissions.setId(globusResponse.getString("access_id"));
                     monitorTemporaryPermissions(permissions.getId(), dataset.getId());
                     logger.info("Access rule " + permissions.getId() + " was created successfully");
-                    
-                    String driverId = dataset.getEffectiveStorageDriverId();
-                    JsonObjectBuilder paths = Json.createObjectBuilder();
-                    for(int i=0;i<numberOfPaths;i++) {
-                        String storageIdentifier = DataAccess.getNewStorageIdentifier(driverId);
-                        int lastIndex = Math.max(storageIdentifier.lastIndexOf("/"), storageIdentifier.lastIndexOf(":")); 
-                        paths.add(storageIdentifier, endpoint.getBasePath() + "/" + storageIdentifier.substring(lastIndex + 1));
-                    
-                    }
-                    response.add("paths", paths.build());
-                    
                 } else {
-                    //Shouldn't happen!
+                    // Shouldn't happen!
                     logger.warning("Access rule id not returned for dataset " + dataset.getId());
                 }
             }
-            } catch (MalformedURLException ex) {
-                //Misconfiguration
-                logger.warning("Failed to create access rule URL for " + endpoint.getId());
-                response.add("status", 500);
-            }
-            return response.build();
+            return result.status;
+        } catch (MalformedURLException ex) {
+            // Misconfiguration
+            logger.warning("Failed to create access rule URL for " + endpoint.getId());
+            return 500;
+        }
     }
 
     public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray referencedFiles) {
@@ -305,26 +303,24 @@ public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray ref
                         "Referenced file " + referencedFile + " is not in an allowed endpoint/path");
             }
             String storageIdentifier = DataAccess.getNewStorageIdentifier(driverId);
-            fileMap.add(referencedFile,
-                    storageIdentifier + "//" + referencedFile);
+            fileMap.add(referencedFile, storageIdentifier + "//" + referencedFile);
         });
         return fileMap.build();
     }
 
-    //Single cache of open rules/permission requests
+    // Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
-            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
-            .scheduler(Scheduler.systemScheduler())
-            .evictionListener((ruleId, datasetId, cause) -> {
-                //Delete rules that expire
+            .expireAfterWrite(
+                    Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
+            .scheduler(Scheduler.systemScheduler()).evictionListener((ruleId, datasetId, cause) -> {
+                // Delete rules that expire
                 logger.info("Rule " + ruleId + " expired");
                 Dataset dataset = datasetSvc.find(datasetId);
                 deletePermission((String) ruleId, dataset, logger);
-              })
-            
+            })
+
             .build();
-    
-    
+
     private void monitorTemporaryPermissions(String ruleId, long datasetId) {
         logger.info("Adding rule " + ruleId + " for dataset " + datasetId);
         rulesCache.put(ruleId, datasetId);
@@ -349,8 +345,7 @@ public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger
 
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId);
 
-        MakeRequestResponse result = makeRequest(url, "Bearer",
-                accessToken, "GET", null);
+        MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "GET", null);
 
         GlobusTask task = null;
 
@@ -540,59 +535,81 @@ static class MakeRequestResponse {
 
     }
 
-    /* unused - may be needed for S3 case
-    private MakeRequestResponse findDirectory(String directory, String clientToken, String globusEndpoint)
-            throws MalformedURLException {
-        URL url = new URL(" https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint + "/ls?path="
-                + directory + "/");
-
-        MakeRequestResponse result = makeRequest(url, "Bearer",
-                clientToken, "GET", null);
-        logger.info("find directory status:" + result.status);
-
-        return result;
-    }
-*/
-    
     /*
-    public boolean giveGlobusPublicPermissions(Dataset dataset)
-            throws UnsupportedEncodingException, MalformedURLException {
-
-        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+     * unused - may be needed for S3 case private MakeRequestResponse
+     * findDirectory(String directory, String clientToken, String globusEndpoint)
+     * throws MalformedURLException { URL url = new
+     * URL(" https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint
+     * + "/ls?path=" + directory + "/");
+     * 
+     * MakeRequestResponse result = makeRequest(url, "Bearer", clientToken, "GET",
+     * null); logger.info("find directory status:" + result.status);
+     * 
+     * return result; }
+     */
 
+    /*
+     * public boolean giveGlobusPublicPermissions(Dataset dataset) throws
+     * UnsupportedEncodingException, MalformedURLException {
+     * 
+     * GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+     * 
+     * 
+     * MakeRequestResponse status = findDirectory(endpoint.getBasePath(),
+     * endpoint.getClientToken(), endpoint.getId());
+     * 
+     * if (status.status == 200) {
+     * 
+     * int perStatus = givePermission("all_authenticated_users", "", "r", dataset);
+     * logger.info("givePermission status " + perStatus); if (perStatus == 409) {
+     * logger.info("Permissions already exist or limit was reached"); } else if
+     * (perStatus == 400) { logger.info("No directory in Globus"); } else if
+     * (perStatus != 201 && perStatus != 200) {
+     * logger.info("Cannot give read permission"); return false; }
+     * 
+     * } else if (status.status == 404) {
+     * logger.info("There is no globus directory"); } else {
+     * logger.severe("Cannot find directory in globus, status " + status); return
+     * false; }
+     * 
+     * return true; }
+     */
 
-        MakeRequestResponse status = findDirectory(endpoint.getBasePath(), endpoint.getClientToken(), endpoint.getId());
+    // Single cache of open rules/permission requests
+    private final Cache<String, JsonObject> downloadCache = Caffeine.newBuilder()
+            .expireAfterWrite(
+                    Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) + 300, ChronoUnit.MINUTES))
+            .scheduler(Scheduler.systemScheduler()).evictionListener((downloadId, datasetId, cause) -> {
+                // Delete downloads that expire
+                logger.info("Download for " + downloadId + " expired");
+            })
 
-        if (status.status == 200) {
+            .build();
+    
+    public JsonObject getFilesForDownload(String downloadId) {
+        return downloadCache.getIfPresent(downloadId);
+    }
 
-            int perStatus = givePermission("all_authenticated_users", "", "r", dataset);
-            logger.info("givePermission status " + perStatus);
-            if (perStatus == 409) {
-                logger.info("Permissions already exist or limit was reached");
-            } else if (perStatus == 400) {
-                logger.info("No directory in Globus");
-            } else if (perStatus != 201 && perStatus != 200) {
-                logger.info("Cannot give read permission");
-                return false;
-            }
+    public int setPermissionForDownload(Dataset dataset, String principal) {
+        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+        String principalType = "identity";
 
-        } else if (status.status == 404) {
-            logger.info("There is no globus directory");
-        } else {
-            logger.severe("Cannot find directory in globus, status " + status);
-            return false;
-        }
+        Permissions permissions = new Permissions();
+        permissions.setDATA_TYPE("access");
+        permissions.setPrincipalType(principalType);
+        permissions.setPrincipal(principal);
+        permissions.setPath(endpoint.getBasePath() + "/");
+        permissions.setPermissions("r");
 
-        return true;
+        return requestPermission(endpoint, dataset, permissions);
     }
-*/
     
     // Generates the URL to launch the Globus app
     public String getGlobusAppUrlForDataset(Dataset d) {
         return getGlobusAppUrlForDataset(d, true, null);
     }
 
-    public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df) {
+    public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<FileMetadata> fileMetadataList) {
         String localeCode = session.getLocaleCode();
         ApiToken apiToken = null;
         User user = session.getUser();
@@ -610,49 +627,60 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
         } catch (Exception e) {
             logger.warning("GlobusAppUrlForDataset: Failed to get storePrefix for " + driverId);
         }
+
         // Use URLTokenUtil for params currently in common with external tools.
-        URLTokenUtil tokenUtil = new URLTokenUtil(d, df, apiToken, localeCode);
-        String appUrl;
+        URLTokenUtil tokenUtil = new URLTokenUtil(d, null, apiToken, localeCode);
+        String appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost");
+        String callback = null;
         if (upload) {
-            appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
-                    + "/upload?dvLocale={localeCode}";
-            String callback = SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + d.getId()
+            appUrl = appUrl + "/upload?dvLocale={localeCode}";
+            callback = SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + d.getId()
                     + "/globusUploadParameters?locale=" + localeCode;
-            if (apiToken != null) {
-                callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(),
-                        HttpMethod.GET,
-                        JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString());
-            } else {
-                // Shouldn't happen
-                logger.warning("unable to get api token for user: " + user.getIdentifier());
-            }
-            appUrl = appUrl + "&callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback));
         } else {
-            if (df == null) {
-                appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
-                        + "/download?datasetPid={datasetPid}&siteUrl={siteUrl}"
-                        + ((apiToken != null) ? "&apiToken={apiToken}" : "")
-                        + "&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}";
-            } else {
-                String rawStorageId = df.getStorageIdentifier();
-                rawStorageId=rawStorageId.substring(rawStorageId.lastIndexOf(":")+1);
-                appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
-                        + "/download-file?datasetPid={datasetPid}&siteUrl={siteUrl}"
-                        + ((apiToken != null) ? "&apiToken={apiToken}" : "")
-                        + "&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}&fileId={fileId}&storageIdentifier="
-                        + rawStorageId + "&fileName=" + df.getCurrentName();
+            // Download
+            ArrayList<DataFile> dataFiles = new ArrayList<DataFile>(fileMetadataList.size());
+            for (FileMetadata fileMetadata : fileMetadataList) {
+                dataFiles.add(fileMetadata.getDataFile());
             }
+            JsonObject files = getFilesMap(dataFiles, d);
+
+            String downloadId = UUID.randomUUID().toString();
+            downloadCache.put(downloadId, files);
+            appUrl = appUrl + "/download?dvLocale={localeCode}";
+            callback = SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + d.getId()
+                    + "/globusDownloadParameters?locale=" + localeCode + "&downloadId=" + downloadId;
+
         }
+        if (apiToken != null) {
+            callback = UrlSignerUtil.signUrl(callback, 5, apiToken.getAuthenticatedUser().getUserIdentifier(),
+                    HttpMethod.GET,
+                    JvmSettings.API_SIGNING_SECRET.lookupOptional().orElse("") + apiToken.getTokenString());
+        } else {
+            // Shouldn't happen
+            logger.warning("Unable to get api token for user: " + user.getIdentifier());
+        }
+        appUrl = appUrl + "&callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback));
+
         String finalUrl = tokenUtil.replaceTokensWithValues(appUrl);
         logger.info("Calling app: " + finalUrl);
         return finalUrl;
     }
 
-    public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken) {
-        return URLTokenUtil.getScriptForUrl(getGlobusAppUrlForDataset(dataset, false, null));
-        
+    public JsonObject getFilesMap(ArrayList<DataFile> dataFiles, Dataset d) {
+        JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
+        for(DataFile df: dataFiles) {
+            String fileLocation = DataAccess
+                    .getLocationFromStorageId(df.getStorageIdentifier(), d);
+            filesBuilder.add(df.getId().toString(), fileLocation);
+        }
+        return filesBuilder.build();
     }
-    
+
+    public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List<FileMetadata> downloadFMList) {
+        return URLTokenUtil.getScriptForUrl(getGlobusAppUrlForDataset(dataset, false, downloadFMList));
+
+    }
+
     @Asynchronous
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
     public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, String httpRequestUrl,
@@ -749,7 +777,8 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                 JsonArray filesJsonArray = jsonData.getJsonArray("files");
 
                 if (filesJsonArray != null) {
-                    String datasetIdentifier = dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
+                    String datasetIdentifier = dataset.getAuthorityForFileStorage() + "/"
+                            + dataset.getIdentifierForFileStorage();
 
                     for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) {
 
@@ -758,16 +787,16 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                         String storageIdentifier = fileJsonObject.getString("storageIdentifier");
                         String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier);
                         String storeId = parts[0];
-                        //If this is an S3 store, we need to split out the bucket name
+                        // If this is an S3 store, we need to split out the bucket name
                         String[] bits = parts[1].split(":");
                         String bucketName = "";
-                        if(bits.length > 1) {
+                        if (bits.length > 1) {
                             bucketName = bits[0];
                         }
                         String fileId = bits[bits.length - 1];
 
                         // fullpath s3://gcs5-bucket1/10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873
-                        //or globus:///10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873
+                        // or globus:///10.5072/FK2/3S6G2E/1781cfeb8a7-4ad9418a5873
                         String fullPath = storeId + "://" + bucketName + "/" + datasetIdentifier + "/" + fileId;
                         String fileName = fileJsonObject.getString("fileName");
 
@@ -777,8 +806,8 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                     // calculateMissingMetadataFields: checksum, mimetype
                     JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, globusLogger);
                     JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files");
-logger.info("Size: " + newfilesJsonArray.size());
-logger.info("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
+                    logger.info("Size: " + newfilesJsonArray.size());
+                    logger.info("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
                     JsonArrayBuilder jsonDataSecondAPI = Json.createArrayBuilder();
 
                     for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) {
@@ -787,31 +816,32 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                         String storageIdentifier = fileJsonObject.getString("storageIdentifier");
                         String fileName = fileJsonObject.getString("fileName");
                         String[] parts = DataAccess.getDriverIdAndStorageLocation(storageIdentifier);
-                        //If this is an S3 store, we need to split out the bucket name
+                        // If this is an S3 store, we need to split out the bucket name
                         String[] bits = parts[1].split(":");
-                        if(bits.length > 1) {
+                        if (bits.length > 1) {
                         }
                         String fileId = bits[bits.length - 1];
-                        
+
                         List<JsonObject> newfileJsonObject = IntStream.range(0, newfilesJsonArray.size())
                                 .mapToObj(index -> ((JsonObject) newfilesJsonArray.get(index)).getJsonObject(fileId))
                                 .filter(Objects::nonNull).collect(Collectors.toList());
                         if (newfileJsonObject != null) {
                             logger.info("List Size: " + newfileJsonObject.size());
-                            //if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) {
-                                JsonPatch path = Json.createPatchBuilder()
-                                        .add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build();
-                                fileJsonObject = path.apply(fileJsonObject);
-                                path = Json.createPatchBuilder()
-                                        .add("/mimeType", newfileJsonObject.get(0).getString("mime")).build();
-                                fileJsonObject = path.apply(fileJsonObject);
-                                jsonDataSecondAPI.add(fileJsonObject);
-                                countSuccess++;
-                           // } else {
-                           //     globusLogger.info(fileName
-                           //             + " will be skipped from adding to dataset by second API due to missing values ");
-                           //     countError++;
-                           // }
+                            // if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) {
+                            JsonPatch path = Json.createPatchBuilder()
+                                    .add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build();
+                            fileJsonObject = path.apply(fileJsonObject);
+                            path = Json.createPatchBuilder()
+                                    .add("/mimeType", newfileJsonObject.get(0).getString("mime")).build();
+                            fileJsonObject = path.apply(fileJsonObject);
+                            jsonDataSecondAPI.add(fileJsonObject);
+                            countSuccess++;
+                            // } else {
+                            // globusLogger.info(fileName
+                            // + " will be skipped from adding to dataset by second API due to missing
+                            // values ");
+                            // countError++;
+                            // }
                         } else {
                             globusLogger.info(fileName
                                     + " will be skipped from adding to dataset by second API due to missing values ");
@@ -828,8 +858,9 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                             + datasetIdentifier + " -F jsonData='" + newjsonData + "'";
                     System.out.println("*******====command ==== " + command);
 
-                    //ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of calling API
-                
+                    // ToDo - refactor to call AddReplaceFileHelper.addFiles directly instead of
+                    // calling API
+
                     String output = addFilesAsync(command, globusLogger);
                     if (output.equalsIgnoreCase("ok")) {
                         // if(!taskSkippedFiles)
@@ -856,8 +887,6 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                 globusLogger.info("Files failures: " + countError.toString());
                 globusLogger.info("Finished upload via Globus job.");
 
-
-
             } catch (Exception e) {
                 logger.info("Exception from globusUpload call ");
                 e.printStackTrace();
@@ -917,11 +946,11 @@ private String addFiles(String curlCommand, Logger globusLogger) {
 
             status = jsonObject.getString("status");
         } catch (Exception ex) {
-            if(ex instanceof JsonParsingException) {
+            if (ex instanceof JsonParsingException) {
                 globusLogger.log(Level.SEVERE, "Error parsing dataset json.");
-            } else { 
+            } else {
                 globusLogger.log(Level.SEVERE,
-                    "******* Unexpected Exception while executing api/datasets/:persistentId/add call ", ex);
+                        "******* Unexpected Exception while executing api/datasets/:persistentId/add call ", ex);
             }
         }
 
@@ -1004,11 +1033,13 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
 
     Executor executor = Executors.newFixedThreadPool(10);
 
-    private GlobusTask globusStatusCheck(GlobusEndpoint endpoint, String taskId, Logger globusLogger) throws MalformedURLException {
+    private GlobusTask globusStatusCheck(GlobusEndpoint endpoint, String taskId, Logger globusLogger)
+            throws MalformedURLException {
         boolean taskCompletion = false;
         String status = "";
         GlobusTask task = null;
-        int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault(settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 50);
+        int pollingInterval = SystemConfig.getIntLimitFromStringOrDefault(
+                settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusPollingInterval), 50);
         do {
             try {
                 globusLogger.info("checking globus transfer task   " + taskId);
@@ -1139,8 +1170,8 @@ private FileDetailsHolder calculateDetails(String id, Logger globusLogger)
             } catch (IOException ioex) {
                 count = 3;
                 logger.info(ioex.getMessage());
-                globusLogger.info("DataFile (fullPath " + fullPath
-                        + ") does not appear to be accessible within Dataverse: ");
+                globusLogger.info(
+                        "DataFile (fullPath " + fullPath + ") does not appear to be accessible within Dataverse: ");
             } catch (Exception ex) {
                 count = count + 1;
                 ex.printStackTrace();
@@ -1167,7 +1198,7 @@ public String calculatemime(String fileName) throws InterruptedException {
         String finalType = FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
         String type = FileUtil.determineFileTypeByNameAndExtension(fileName);
 
-        if (type!=null && !type.isBlank()) {
+        if (type != null && !type.isBlank()) {
             if (FileUtil.useRecognizedType(finalType, type)) {
                 finalType = type;
             }
@@ -1365,7 +1396,7 @@ public String calculatemime(String fileName) throws InterruptedException {
      * updatePermision(clientTokenUser, directory, "identity", "r"); return true; }
      * 
      */
-    
+
     GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         Dataset dataset = null;
         if (dvObject instanceof Dataset) {
@@ -1377,35 +1408,37 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         }
         String driverId = dataset.getEffectiveStorageDriverId();
         GlobusEndpoint endpoint = null;
-        
+
         String directoryPath = GlobusAccessibleStore.getTransferPath(driverId);
 
-        if (GlobusAccessibleStore.isDataverseManaged(driverId) && (dataset!=null)) {
-            directoryPath = directoryPath + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage();
+        if (GlobusAccessibleStore.isDataverseManaged(driverId) && (dataset != null)) {
+            directoryPath = directoryPath + "/" + dataset.getAuthorityForFileStorage() + "/"
+                    + dataset.getIdentifierForFileStorage();
             logger.info("directoryPath now: " + directoryPath);
 
         } else {
-            //remote store - may have path in file storageidentifier
-            String relPath = dvObject.getStorageIdentifier().substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
+            // remote store - may have path in file storageidentifier
+            String relPath = dvObject.getStorageIdentifier()
+                    .substring(dvObject.getStorageIdentifier().lastIndexOf("//") + 2);
             int filenameStart = relPath.lastIndexOf("/") + 1;
             if (filenameStart > 0) {
                 directoryPath = directoryPath + relPath.substring(0, filenameStart);
             }
         }
         logger.info("directoryPath finally: " + directoryPath);
-        
-        String endpointId = GlobusAccessibleStore.getTransferPath(driverId);
-        
+
+        String endpointId = GlobusAccessibleStore.getTransferEndpointId(driverId);
+
         logger.info("endpointId: " + endpointId);
-        
+
         String globusToken = GlobusAccessibleStore.getGlobusToken(driverId);
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         String clientToken = accessToken.getOtherTokens().get(0).getAccessToken();
-logger.info("clientToken: " + clientToken);
+        logger.info("clientToken: " + clientToken);
         endpoint = new GlobusEndpoint(endpointId, clientToken, directoryPath);
 
         return endpoint;
     }
-    
+
 }

From 7fd2bb2ad30eb0bdeb26083089d5788abb95cbeb Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:18:43 -0500
Subject: [PATCH 774/922] convenience method

---
 .../iq/dataverse/dataaccess/GlobusAccessibleStore.java        | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index ce75395c883..aad1dab5eab 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -14,6 +14,10 @@ public static boolean isDataverseManaged(String driverId) {
         return Boolean.parseBoolean(StorageIO.getConfigParamForDriver(driverId, MANAGED));
     }
     
+    public static String getTransferEnpointWithPath(String driverId) {
+        return StorageIO.getConfigParamForDriver(driverId, GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH);
+    }
+    
     public static String getTransferEndpointId(String driverId) {
         String endpointWithBasePath = StorageIO.getConfigParamForDriver(driverId, TRANSFER_ENDPOINT_WITH_BASEPATH);
         int pathStart = endpointWithBasePath.indexOf("/");

From 4774541a3e1ab766c834e666937b417260d68705 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:19:08 -0500
Subject: [PATCH 775/922] adapt to changed method signature

---
 .../edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
index d645dc3307f..cc064976982 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
@@ -22,6 +22,7 @@
 import jakarta.ws.rs.ext.Provider;
 
 import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.FileMetadata;
 import edu.harvard.iq.dataverse.dataaccess.*;
 import edu.harvard.iq.dataverse.datavariable.DataVariable;
 import edu.harvard.iq.dataverse.engine.command.Command;
@@ -212,7 +213,9 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
                         if (di.getConversionParam().equals("format")) {
 
                             if ("GlobusTransfer".equals(di.getConversionParamValue())) {
-                                redirect_url_str = globusService.getGlobusAppUrlForDataset(dataFile.getOwner(), false, dataFile);
+                                List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
+                                downloadFMList.add(dataFile.getFileMetadata());
+                                redirect_url_str = globusService.getGlobusAppUrlForDataset(dataFile.getOwner(), false, downloadFMList);
                             }
                         }
                     }

From 55c78433b1c3f876081ba51951ff31012b72fee1 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:19:58 -0500
Subject: [PATCH 776/922] use rules cache timout in upload, add download
 methods

---
 .../harvard/iq/dataverse/api/Datasets.java    | 244 +++++++++++++++++-
 1 file changed, 238 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index a57f373f106..2d2db1fad7d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3599,17 +3599,21 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         } else {
             params.add("referenceEndpointsWithPaths", referenceEndpointsWithPaths);
         }
-
+        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) * 60;
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         String requestCallName = managed ? "requestGlobusTransferPaths" : "requestGlobusReferencePaths";
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, requestCallName)
                     .add(URLTokenUtil.HTTP_METHOD, "POST")
-                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusPaths")
-                    .add(URLTokenUtil.TIMEOUT, 300));
+                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusUploadPaths")
+                    .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addGlobusFiles")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
-                .add(URLTokenUtil.TIMEOUT, 300));
+                .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        allowedApiCalls.add(
+                Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata").add(URLTokenUtil.HTTP_METHOD, "GET")
+                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
+                        .add(URLTokenUtil.TIMEOUT, 300));
         allowedApiCalls.add(
                 Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
                         .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
@@ -3630,7 +3634,7 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
      */
     @POST
     @AuthRequired
-    @Path("{id}/requestGlobusPaths")
+    @Path("{id}/requestGlobusUploadPaths")
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
     public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, String jsonBody
@@ -3709,6 +3713,234 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
         }
 
     }
+    
+    /**
+     * Retrieve the parameters and signed URLs required to perform a globus
+     * transfer/download. This api endpoint is expected to be called as a signed callback
+     * after the globus-dataverse app/other app is launched, but it will accept
+     * other forms of authentication.
+     * 
+     * @param crc
+     * @param datasetId
+     */
+    @GET
+    @AuthRequired
+    @Path("{id}/globusDownloadParameters")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @QueryParam(value = "locale") String locale, @QueryParam(value = "downloadId") String downloadId)
+    {
+        // -------------------------------------
+        // (1) Get the user from the ContainerRequestContext
+        // -------------------------------------
+        AuthenticatedUser authUser;
+        try {
+            authUser = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse e) {
+            return e.getResponse();
+        }
+        // -------------------------------------
+        // (2) Get the Dataset Id
+        // -------------------------------------
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(datasetId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        String storeId = dataset.getEffectiveStorageDriverId();
+        //acceptsGlobusTransfers should only be true for an S3 or globus store
+        if(!(GlobusAccessibleStore.acceptsGlobusTransfers(storeId) || GlobusAccessibleStore.allowsGlobusReferences(storeId))) {
+            return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
+        }
+
+        JsonObject files = globusService.getFilesForDownload(downloadId);
+        if(files==null) {
+            return notFound(BundleUtil.getStringFromBundle("datasets.api.globusdownloadnotfound"));
+        }
+        
+        URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
+
+        boolean managed = GlobusAccessibleStore.isDataverseManaged(storeId);
+        String transferEndpoint = null;
+        
+        
+        JsonObjectBuilder queryParams = Json.createObjectBuilder();
+        queryParams.add("queryParameters",
+                Json.createArrayBuilder().add(Json.createObjectBuilder().add("datasetId", "{datasetId}"))
+                        .add(Json.createObjectBuilder().add("siteUrl", "{siteUrl}"))
+                        .add(Json.createObjectBuilder().add("datasetVersion", "{datasetVersion}"))
+                        .add(Json.createObjectBuilder().add("dvLocale", "{localeCode}"))
+                        .add(Json.createObjectBuilder().add("datasetPid", "{datasetPid}")));
+        JsonObject substitutedParams = tokenUtil.getParams(queryParams.build());
+        JsonObjectBuilder params = Json.createObjectBuilder();
+        substitutedParams.keySet().forEach((key) -> {
+            params.add(key, substitutedParams.get(key));
+        });
+        params.add("managed", Boolean.toString(managed));
+        if(managed) {
+            transferEndpoint = GlobusAccessibleStore.getTransferEndpointId(storeId);
+            params.add("endpoint", transferEndpoint);
+        }
+        params.add("files", files);
+        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) * 60;
+        JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "monitorGlobusDownload")
+                    .add(URLTokenUtil.HTTP_METHOD, "POST")
+                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/monitorGlobusDownload")
+                    .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusDownload")
+                .add(URLTokenUtil.HTTP_METHOD, "POST")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusDownload?downloadId=" + downloadId)
+                .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        allowedApiCalls.add(
+                Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata").add(URLTokenUtil.HTTP_METHOD, "GET")
+                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
+                        .add(URLTokenUtil.TIMEOUT, 300));
+        allowedApiCalls.add(
+                Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
+                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
+                        .add(URLTokenUtil.TIMEOUT, 300));
+
+        return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
+    }
+    
+    /** Requests permissions for a given globus user to download the specified files the dataset
+     * 
+     * @param crc
+     * @param datasetId
+     * @param jsonData
+     * @return
+     * @throws IOException
+     * @throws ExecutionException
+     * @throws InterruptedException
+     */
+    @POST
+    @AuthRequired
+    @Path("{id}/requestGlobusDownloadPaths")
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response requestGlobusDownload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,
+            @QueryParam(value = "downloadId") String downloadId, String jsonBody)
+            throws IOException, ExecutionException, InterruptedException {
+
+        logger.info(" ====  (api allowGlobusDownload) jsonBody   ====== " + jsonBody);
+
+        if (!systemConfig.isGlobusDownload()) {
+            return error(Response.Status.SERVICE_UNAVAILABLE,
+                    BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
+        }
+
+        // -------------------------------------
+        // (1) Get the user from the ContainerRequestContext
+        // -------------------------------------
+        User user = getRequestUser(crc);
+
+        // -------------------------------------
+        // (2) Get the Dataset Id
+        // -------------------------------------
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(datasetId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        JsonObject body = JsonUtil.getJsonObject(jsonBody);
+        Set<String> fileIds = null;
+        if (downloadId != null) {
+            JsonObject files = globusService.getFilesForDownload(downloadId);
+            if (files != null) {
+                fileIds = files.keySet();
+            }
+        } else {
+            if (body.containsKey("fileIds")) {
+                Collection<JsonValue> fileVals = body.getJsonArray("fileIds").getValuesAs(JsonValue.class);
+                fileIds = new HashSet<String>(fileVals.size());
+                for (JsonValue fileVal : fileVals) {
+                    String id = null;
+                    switch (fileVal.getValueType()) {
+                    case STRING:
+                        id = ((JsonString) fileVal).getString();
+                        break;
+                    case NUMBER:
+                        id = ((JsonNumber) fileVal).toString();
+                        break;
+                    default:
+                        return badRequest("fileIds must be numeric or string (ids/PIDs)");
+                    }
+                    ;
+                    fileIds.add(id);
+                }
+            } else {
+                return badRequest("fileIds JsonArray of file ids/PIDs required in POST body");
+            }
+        }
+
+        if (fileIds.isEmpty()) {
+            return notFound(BundleUtil.getStringFromBundle("datasets.api.globusdownloadnotfound"));
+        }
+        ArrayList<DataFile> dataFiles = new ArrayList<DataFile>(fileIds.size());
+        for (String id : fileIds) {
+            boolean published = false;
+
+            DataFile df = null;
+            try {
+                df = findDataFileOrDie(id);
+            } catch (WrappedResponse wr) {
+                return wr.getResponse();
+            }
+            if (df.getOwner() != dataset) {
+                return badRequest("All files must be in the dataset");
+            }
+            dataFiles.add(df);
+
+            for (FileMetadata fm : df.getFileMetadatas()) {
+                if (fm.getDatasetVersion().isPublished()) {
+                    published = true;
+                    break;
+                }
+            }
+
+            if (!published) {
+                // If the file is not published, they can still download the file, if the user
+                // has the permission to view unpublished versions:
+
+                if (!permissionService.hasPermissionsFor(user, df.getOwner(),
+                        EnumSet.of(Permission.ViewUnpublishedDataset))) {
+                    return forbidden("User doesn't have permission to download file: " + id);
+                }
+            } else { // published and restricted and/or embargoed
+                if (df.isRestricted() || FileUtil.isActivelyEmbargoed(df))
+                    // This line also handles all three authenticated session user, token user, and
+                    // guest cases.
+                    if (!permissionService.hasPermissionsFor(user, df, EnumSet.of(Permission.DownloadFile))) {
+                        return forbidden("User doesn't have permission to download file: " + id);
+                    }
+
+            }
+        }
+        // Allowed to download all requested files
+        JsonObject files = globusService.getFilesMap(dataFiles, dataset);
+        if (GlobusAccessibleStore.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
+            // If managed, give the principal read permissions
+            int status = globusService.setPermissionForDownload(dataset, body.getString("principal"));
+            switch (status) {
+            case 201:
+                return ok(files);
+            case 400:
+                return badRequest("Unable to grant permission");
+            case 409:
+                return conflict("Permission already exists");
+            default:
+                return error(null, "Unexpected error when granting permission");
+            }
+
+        }
+
+        return ok(files);
+    }
 
     /** Monitors a globus download and removes permissions on the dir/dataset when done
      * 
@@ -3722,7 +3954,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
      */
     @POST
     @AuthRequired
-    @Path("{id}/deleteglobusRule")
+    @Path("{id}/monitorGlobusDownload")
     @Consumes(MediaType.MULTIPART_FORM_DATA)
     public Response deleteglobusRule(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,@FormDataParam("jsonData") String jsonData
     ) throws IOException, ExecutionException, InterruptedException {

From e276f1f89c9ca4459f50691051cbff10a3c1ee67 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:20:14 -0500
Subject: [PATCH 777/922] minor text changes

---
 .../dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
index 8adaf746210..9de6bf69832 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
@@ -222,11 +222,11 @@ public String getStorageLocation() throws IOException {
                     .substring(fullStorageLocation.lastIndexOf(DataAccess.SEPARATOR) + DataAccess.SEPARATOR.length());
         }
         if (this.getDvObject() instanceof Dataset) {
-            throw new IOException("RemoteOverlayAccessIO: Datasets are not a supported dvObject");
+            throw new IOException("AbstractRemoteOverlayAccessIO: Datasets are not a supported dvObject");
         } else if (this.getDvObject() instanceof DataFile) {
             fullStorageLocation = StorageIO.getDriverPrefix(this.driverId) + fullStorageLocation;
         } else if (dvObject instanceof Dataverse) {
-            throw new IOException("RemoteOverlayAccessIO: Dataverses are not a supported dvObject");
+            throw new IOException("AbstractRemoteOverlayAccessIO: Dataverses are not a supported dvObject");
         }
         logger.fine("fullStorageLocation: " + fullStorageLocation);
         return fullStorageLocation;

From b9317e885249b10d1661f9d8e2e2535e05bb4737 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:20:39 -0500
Subject: [PATCH 778/922] use new method signature

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index fc18257196d..4b99464504c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -6261,7 +6261,13 @@ public void startGlobusTransfer() {
             apiToken = new ApiToken();
             apiToken.setTokenString(privUrl.getToken());
         }
-        PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken));
+        if(fileMetadataForAction!=null) {
+            List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
+            downloadFMList.add(fileMetadataForAction);
+            PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
+        } else {
+            PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, getSelectedDownloadableFiles()));
+        }
     }
 
     public String getWebloaderUrlForDataset(Dataset d) {

From a31aa074ebd600dc6dbdd51916388ca45f199280 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 21 Nov 2023 16:20:55 -0500
Subject: [PATCH 779/922] add not found message

---
 src/main/java/propertyFiles/Bundle.properties | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index ea456f54907..693eb7ca294 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -2652,6 +2652,7 @@ datasets.api.version.files.invalid.access.status=Invalid access status: {0}
 datasets.api.deaccessionDataset.invalid.version.identifier.error=Only {0} or a specific version can be deaccessioned
 datasets.api.deaccessionDataset.invalid.forward.url=Invalid deaccession forward URL: {0}
 datasets.api.globusdownloaddisabled=File transfer from Dataverse via Globus is not available for this dataset.
+datasets.api.globusdownloadnotfound=List of files to transfer not found.
 datasets.api.globusuploaddisabled=File transfer to Dataverse via Globus is not available for this dataset.
 
 #Dataverses.java

From 0ea4615174640b2d495a92cb064d53850fef6265 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 22 Nov 2023 14:40:13 -0500
Subject: [PATCH 780/922] fix dataset page access menu globus transfer option

---
 src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 4b99464504c..ab7e6a8554a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -6266,7 +6266,13 @@ public void startGlobusTransfer() {
             downloadFMList.add(fileMetadataForAction);
             PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
         } else {
-            PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, getSelectedDownloadableFiles()));
+            if(getSelectedDownloadableFiles()!=null) {
+                PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, getSelectedDownloadableFiles()));
+            } else {
+                //ToDo: For non-public, need the subset that are downloadable by the user
+                //ToDo: For mixed (some in backing store), need the ones in the globus store
+                PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, workingVersion.getFileMetadatas()));
+            }
         }
     }
 

From cf5cffce1b34229f2264fb020787229187e652c3 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 22 Nov 2023 15:02:30 -0500
Subject: [PATCH 781/922] multiple download fixes, reformat globus code

---
 .../harvard/iq/dataverse/api/Datasets.java    | 122 +++++++++---------
 1 file changed, 62 insertions(+), 60 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 2d2db1fad7d..f1bfc0ed69e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3541,10 +3541,9 @@ public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
     @GET
     @AuthRequired
     @Path("{id}/globusUploadParameters")
-    @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
-    public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @QueryParam(value = "locale") String locale)
-    {
+    public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,
+            @QueryParam(value = "locale") String locale) {
         // -------------------------------------
         // (1) Get the user from the ContainerRequestContext
         // -------------------------------------
@@ -3565,17 +3564,18 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
             return wr.getResponse();
         }
         String storeId = dataset.getEffectiveStorageDriverId();
-        //acceptsGlobusTransfers should only be true for an S3 or globus store
-        if(!GlobusAccessibleStore.acceptsGlobusTransfers(storeId) && !GlobusAccessibleStore.allowsGlobusReferences(storeId)) {
+        // acceptsGlobusTransfers should only be true for an S3 or globus store
+        if (!GlobusAccessibleStore.acceptsGlobusTransfers(storeId)
+                && !GlobusAccessibleStore.allowsGlobusReferences(storeId)) {
             return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled"));
         }
-        
+
         URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
 
         boolean managed = GlobusAccessibleStore.isDataverseManaged(storeId);
         String transferEndpoint = null;
         JsonArray referenceEndpointsWithPaths = null;
-        if(managed) {
+        if (managed) {
             transferEndpoint = GlobusAccessibleStore.getTransferEndpointId(storeId);
         } else {
             referenceEndpointsWithPaths = GlobusAccessibleStore.getReferenceEndpointsWithPaths(storeId);
@@ -3594,7 +3594,7 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
             params.add(key, substitutedParams.get(key));
         });
         params.add("managed", Boolean.toString(managed));
-        if(transferEndpoint!= null) {
+        if (transferEndpoint != null) {
             params.add("endpoint", transferEndpoint);
         } else {
             params.add("referenceEndpointsWithPaths", referenceEndpointsWithPaths);
@@ -3602,18 +3602,18 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) * 60;
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         String requestCallName = managed ? "requestGlobusTransferPaths" : "requestGlobusReferencePaths";
-        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, requestCallName)
-                    .add(URLTokenUtil.HTTP_METHOD, "POST")
-                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusUploadPaths")
-                    .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        allowedApiCalls.add(
+                Json.createObjectBuilder().add(URLTokenUtil.NAME, requestCallName).add(URLTokenUtil.HTTP_METHOD, "POST")
+                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusUploadPaths")
+                        .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addGlobusFiles")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
                 .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
-        allowedApiCalls.add(
-                Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata").add(URLTokenUtil.HTTP_METHOD, "GET")
-                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
-                        .add(URLTokenUtil.TIMEOUT, 300));
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata")
+                .add(URLTokenUtil.HTTP_METHOD, "GET")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
+                .add(URLTokenUtil.TIMEOUT, 300));
         allowedApiCalls.add(
                 Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
                         .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
@@ -3621,8 +3621,9 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
 
         return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
     }
-    
-    /** Requests permissions for a given globus user to upload to the dataset
+
+    /**
+     * Requests permissions for a given globus user to upload to the dataset
      * 
      * @param crc
      * @param datasetId
@@ -3637,15 +3638,14 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
     @Path("{id}/requestGlobusUploadPaths")
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
-    public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, String jsonBody
-    ) throws IOException, ExecutionException, InterruptedException {
-
+    public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,
+            String jsonBody) throws IOException, ExecutionException, InterruptedException {
 
         logger.info(" ====  (api allowGlobusUpload) jsonBody   ====== " + jsonBody);
 
-
         if (!systemConfig.isGlobusUpload()) {
-            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
+            return error(Response.Status.SERVICE_UNAVAILABLE,
+                    BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
         }
 
         // -------------------------------------
@@ -3713,12 +3713,12 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
         }
 
     }
-    
+
     /**
      * Retrieve the parameters and signed URLs required to perform a globus
-     * transfer/download. This api endpoint is expected to be called as a signed callback
-     * after the globus-dataverse app/other app is launched, but it will accept
-     * other forms of authentication.
+     * transfer/download. This api endpoint is expected to be called as a signed
+     * callback after the globus-dataverse app/other app is launched, but it will
+     * accept other forms of authentication.
      * 
      * @param crc
      * @param datasetId
@@ -3726,10 +3726,9 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
     @GET
     @AuthRequired
     @Path("{id}/globusDownloadParameters")
-    @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
-    public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @QueryParam(value = "locale") String locale, @QueryParam(value = "downloadId") String downloadId)
-    {
+    public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,
+            @QueryParam(value = "locale") String locale, @QueryParam(value = "downloadId") String downloadId) {
         // -------------------------------------
         // (1) Get the user from the ContainerRequestContext
         // -------------------------------------
@@ -3750,22 +3749,22 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
             return wr.getResponse();
         }
         String storeId = dataset.getEffectiveStorageDriverId();
-        //acceptsGlobusTransfers should only be true for an S3 or globus store
-        if(!(GlobusAccessibleStore.acceptsGlobusTransfers(storeId) || GlobusAccessibleStore.allowsGlobusReferences(storeId))) {
+        // acceptsGlobusTransfers should only be true for an S3 or globus store
+        if (!(GlobusAccessibleStore.acceptsGlobusTransfers(storeId)
+                || GlobusAccessibleStore.allowsGlobusReferences(storeId))) {
             return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
         }
 
         JsonObject files = globusService.getFilesForDownload(downloadId);
-        if(files==null) {
+        if (files == null) {
             return notFound(BundleUtil.getStringFromBundle("datasets.api.globusdownloadnotfound"));
         }
-        
+
         URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
 
         boolean managed = GlobusAccessibleStore.isDataverseManaged(storeId);
         String transferEndpoint = null;
-        
-        
+
         JsonObjectBuilder queryParams = Json.createObjectBuilder();
         queryParams.add("queryParameters",
                 Json.createArrayBuilder().add(Json.createObjectBuilder().add("datasetId", "{datasetId}"))
@@ -3779,7 +3778,7 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
             params.add(key, substitutedParams.get(key));
         });
         params.add("managed", Boolean.toString(managed));
-        if(managed) {
+        if (managed) {
             transferEndpoint = GlobusAccessibleStore.getTransferEndpointId(storeId);
             params.add("endpoint", transferEndpoint);
         }
@@ -3787,17 +3786,18 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
         int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) * 60;
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "monitorGlobusDownload")
-                    .add(URLTokenUtil.HTTP_METHOD, "POST")
-                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/monitorGlobusDownload")
-                    .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+                .add(URLTokenUtil.HTTP_METHOD, "POST")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/monitorGlobusDownload")
+                .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusDownload")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
-                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusDownload?downloadId=" + downloadId)
+                .add(URLTokenUtil.URL_TEMPLATE,
+                        "/api/v1/datasets/{datasetId}/requestGlobusDownload?downloadId=" + downloadId)
                 .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
-        allowedApiCalls.add(
-                Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata").add(URLTokenUtil.HTTP_METHOD, "GET")
-                        .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
-                        .add(URLTokenUtil.TIMEOUT, 300));
+        allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata")
+                .add(URLTokenUtil.HTTP_METHOD, "GET")
+                .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
+                .add(URLTokenUtil.TIMEOUT, 300));
         allowedApiCalls.add(
                 Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
                         .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
@@ -3805,8 +3805,10 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
 
         return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
     }
-    
-    /** Requests permissions for a given globus user to download the specified files the dataset
+
+    /**
+     * Requests permissions for a given globus user to download the specified files
+     * the dataset
      * 
      * @param crc
      * @param datasetId
@@ -3818,7 +3820,7 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
      */
     @POST
     @AuthRequired
-    @Path("{id}/requestGlobusDownloadPaths")
+    @Path("{id}/requestGlobusDownload")
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
     public Response requestGlobusDownload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,
@@ -3884,6 +3886,7 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
         ArrayList<DataFile> dataFiles = new ArrayList<DataFile>(fileIds.size());
         for (String id : fileIds) {
             boolean published = false;
+            logger.info("File id: " + id);
 
             DataFile df = null;
             try {
@@ -3891,7 +3894,7 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
             } catch (WrappedResponse wr) {
                 return wr.getResponse();
             }
-            if (df.getOwner() != dataset) {
+            if (!df.getOwner().equals(dataset)) {
                 return badRequest("All files must be in the dataset");
             }
             dataFiles.add(df);
@@ -3942,7 +3945,9 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
         return ok(files);
     }
 
-    /** Monitors a globus download and removes permissions on the dir/dataset when done
+    /**
+     * Monitors a globus download and removes permissions on the dir/dataset when
+     * done
      * 
      * @param crc
      * @param datasetId
@@ -3955,16 +3960,15 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
     @POST
     @AuthRequired
     @Path("{id}/monitorGlobusDownload")
-    @Consumes(MediaType.MULTIPART_FORM_DATA)
-    public Response deleteglobusRule(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,@FormDataParam("jsonData") String jsonData
-    ) throws IOException, ExecutionException, InterruptedException {
-
+    @Consumes(MediaType.APPLICATION_JSON)
+    public Response monitorGlobusDownload(@Context ContainerRequestContext crc, @PathParam("id") String datasetId,
+            String jsonData) throws IOException, ExecutionException, InterruptedException {
 
         logger.info(" ====  (api deleteglobusRule) jsonData   ====== " + jsonData);
 
-
         if (!systemConfig.isGlobusDownload()) {
-            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
+            return error(Response.Status.SERVICE_UNAVAILABLE,
+                    BundleUtil.getStringFromBundle("datasets.api.globusdownloaddisabled"));
         }
 
         // -------------------------------------
@@ -3991,7 +3995,6 @@ public Response deleteglobusRule(@Context ContainerRequestContext crc, @PathPara
 
     }
 
-
     /**
      * Add multiple Files to an existing Dataset
      *
@@ -4003,9 +4006,8 @@ public Response deleteglobusRule(@Context ContainerRequestContext crc, @PathPara
     @AuthRequired
     @Path("{id}/addFiles")
     @Consumes(MediaType.MULTIPART_FORM_DATA)
-    public Response addFilesToDataset(@Context ContainerRequestContext crc,
-                                      @PathParam("id") String idSupplied,
-                                      @FormDataParam("jsonData") String jsonData) {
+    public Response addFilesToDataset(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied,
+            @FormDataParam("jsonData") String jsonData) {
 
         if (!systemConfig.isHTTPUpload()) {
             return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled"));

From 513d8b7e9ab55e3d13181732e2169a4cb7116004 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 22 Nov 2023 15:03:36 -0500
Subject: [PATCH 782/922] comment formatting

---
 .../dataaccess/GlobusOverlayAccessIO.java     | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index a37cd23ad2b..7ec1e2f9e73 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -37,13 +37,20 @@
  * Globus Overlay Driver
  * 
  * Remote: StorageIdentifier format:
- * <globusDriverId>://<baseStorageIdentifier>//<relativePath> Storage location:
- * <globusendpointId/basepath>/<relPath> Internal StorageIdentifier format:
- * <globusDriverId>://<baseStorageIdentifier> Storage location:
- * <globusEndpointId/basepath>/<dataset authority>/<dataset
- * identifier>/<baseStorageIdentifier>
+ * <globusDriverId>://<baseStorageIdentifier>//<relativePath> 
+ * 
+ * Storage location:
+ * <globusendpointId/basepath>/<relPath> 
+ * 
+ * Internal StorageIdentifier format:
+ * <globusDriverId>://<baseStorageIdentifier> 
+ * 
+ * Storage location:
+ * <globusEndpointId/basepath>/<dataset authority>/<dataset identifier>/<baseStorageIdentifier>
  *
- * transfer and reference endpoint formats: <globusEndpointId/basePath>
+ * transfer and reference endpoint formats: 
+ * <globusEndpointId/basePath>
+ * 
  * reference endpoints separated by a comma
  * 
  */

From 5ee190515c5fa901455e2cbc9587756ddbc325c9 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 22 Nov 2023 15:04:58 -0500
Subject: [PATCH 783/922] change filemap for download, fix order of permission
 delete/cache remove

---
 .../dataverse/globus/GlobusServiceBean.java   | 95 +++++++++++--------
 1 file changed, 56 insertions(+), 39 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 28173aa0074..37ec1afd4ed 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -184,30 +184,21 @@ private String getRuleId(GlobusEndpoint endpoint, String principal, String permi
      * @param globusLogger - a separate logger instance, may be null
      */
     public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger) {
-
+        globusLogger.info("Start deleting rule " + ruleId + " for dataset " + dataset.getId());
         if (ruleId.length() > 0) {
             if (dataset != null) {
                 GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
                 if (endpoint != null) {
                     String accessToken = endpoint.getClientToken();
-                    if (globusLogger != null) {
-                        globusLogger.info("Start deleting permissions.");
-                    }
+                    globusLogger.info("Start deleting permissions.");
                     try {
                         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + endpoint.getId()
                                 + "/access/" + ruleId);
                         MakeRequestResponse result = makeRequest(url, "Bearer", accessToken, "DELETE", null);
                         if (result.status != 200) {
-                            if (globusLogger != null) {
-                                globusLogger.warning("Cannot delete access rule " + ruleId);
-                            } else {
-                                // When removed due to a cache ejection, we don't have a globusLogger
-                                logger.warning("Cannot delete access rule " + ruleId);
-                            }
+                            globusLogger.warning("Cannot delete access rule " + ruleId);
                         } else {
-                            if (globusLogger != null) {
-                                globusLogger.info("Access rule " + ruleId + " was deleted successfully");
-                            }
+                            globusLogger.info("Access rule " + ruleId + " was deleted successfully");
                         }
                     } catch (MalformedURLException ex) {
                         logger.log(Level.WARNING,
@@ -585,7 +576,7 @@ static class MakeRequestResponse {
             })
 
             .build();
-    
+
     public JsonObject getFilesForDownload(String downloadId) {
         return downloadCache.getIfPresent(downloadId);
     }
@@ -603,7 +594,7 @@ public int setPermissionForDownload(Dataset dataset, String principal) {
 
         return requestPermission(endpoint, dataset, permissions);
     }
-    
+
     // Generates the URL to launch the Globus app
     public String getGlobusAppUrlForDataset(Dataset d) {
         return getGlobusAppUrlForDataset(d, true, null);
@@ -668,9 +659,14 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<FileMeta
 
     public JsonObject getFilesMap(ArrayList<DataFile> dataFiles, Dataset d) {
         JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
-        for(DataFile df: dataFiles) {
-            String fileLocation = DataAccess
-                    .getLocationFromStorageId(df.getStorageIdentifier(), d);
+        for (DataFile df : dataFiles) {
+            String[] fileInfo = DataAccess.getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(df.getStorageIdentifier(), d));
+            String driverId = fileInfo[0];
+            String fileLocation= fileInfo[1]; 
+            if(GlobusAccessibleStore.isDataverseManaged(driverId)) {
+                String endpointWithBasePath = GlobusAccessibleStore.getTransferEnpointWithPath(driverId);
+                fileLocation = endpointWithBasePath + "/" + fileLocation;
+            }
             filesBuilder.add(df.getId().toString(), fileLocation);
         }
         return filesBuilder.build();
@@ -711,29 +707,35 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
             globusLogger = logger;
         }
 
-        Thread.sleep(5000);
-
         logger.fine("json: " + JsonUtil.prettyPrint(jsonData));
 
         String taskIdentifier = jsonData.getString("taskIdentifier");
 
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
-        // globus task status check
-        GlobusTask task = globusStatusCheck(endpoint, taskIdentifier, globusLogger);
-        String taskStatus = getTaskStatus(task);
-
-        globusLogger.info("Starting an globusUpload ");
-
+        GlobusTask task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger);
         String ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
         logger.info("Found rule: " + ruleId);
         if (ruleId != null) {
             Long datasetId = rulesCache.getIfPresent(ruleId);
             if (datasetId != null) {
-
-                // Will delete rule
+                // Will not delete rule
                 rulesCache.invalidate(ruleId);
             }
         }
+        
+        //Wait before first check
+        Thread.sleep(5000);
+        // globus task status check
+        task = globusStatusCheck(endpoint, taskIdentifier, globusLogger);
+        String taskStatus = getTaskStatus(task);
+
+        globusLogger.info("Starting a globusUpload ");
+
+        if (ruleId != null) {
+            // Transfer is complete, so delete rule
+            deletePermission(ruleId, dataset, globusLogger);
+
+        }
 
         // If success, switch to an EditInProgress lock - do this before removing the
         // GlobusUpload lock
@@ -983,7 +985,7 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
             globusLogger = logger;
         }
 
-        globusLogger.info("Starting an globusDownload ");
+        globusLogger.info("Starting a globusDownload ");
 
         JsonObject jsonObject = null;
         try {
@@ -995,19 +997,34 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         }
 
         String taskIdentifier = jsonObject.getString("taskIdentifier");
-        String ruleId = "";
 
-        try {
-            jsonObject.getString("ruleId");
-        } catch (NullPointerException npe) {
-            globusLogger.log(Level.SEVERE, "Error parsing dataset json. No ruleId: {0}", jsonData);
-        }
+        GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+        logger.info("Endpoint path: " + endpoint.getBasePath());
 
+        // If the rules_cache times out, the permission will be deleted. Presumably that
+        // doesn't affect a
         // globus task status check
-        GlobusTask task = globusStatusCheck(null, taskIdentifier, globusLogger);
+        GlobusTask task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger);
+        String ruleId = getRuleId(endpoint, task.getOwner_id(), "r");
+        if (ruleId != null) {
+            logger.info("Found rule: " + ruleId);
+            Long datasetId = rulesCache.getIfPresent(ruleId);
+            if (datasetId != null) {
+                logger.info("Deleting from cache: rule: " + ruleId);
+                // Will not delete rule
+                rulesCache.invalidate(ruleId);
+            }
+        } else {
+            // Something is wrong - the rule should be there (a race with the cache timing
+            // out?)
+            logger.warning("ruleId not found for taskId: " + taskIdentifier);
+        }
+        task = globusStatusCheck(endpoint, taskIdentifier, globusLogger);
         String taskStatus = getTaskStatus(task);
-
-        if (ruleId.length() > 0) {
+        
+        //Transfer is done (success or failure) so delete the rule
+        if (ruleId != null) {
+            logger.info("Deleting: rule: " + ruleId);
             deletePermission(ruleId, dataset, globusLogger);
         }
 
@@ -1087,7 +1104,7 @@ private String getTaskStatus(GlobusTask task) {
         if (task != null) {
             status = task.getStatus();
             if (status != null) {
-                // The task is in progress.
+                // The task is in progress but is not ok or queued
                 if (status.equalsIgnoreCase("ACTIVE")) {
                     status = "FAILED" + "#" + task.getNice_status() + "#" + task.getNice_status_short_description();
                 } else {

From c6ff2c7cefc939bb14ba9a30df2712c7ff264674 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 22 Nov 2023 15:40:38 -0500
Subject: [PATCH 784/922] fix timeouts, use addFiles call for remote reference
 case

---
 .../harvard/iq/dataverse/api/Datasets.java    | 19 +++++++++++++------
 .../dataverse/globus/GlobusServiceBean.java   |  2 +-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index f1bfc0ed69e..7587c1b6e06 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3599,25 +3599,32 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         } else {
             params.add("referenceEndpointsWithPaths", referenceEndpointsWithPaths);
         }
-        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) * 60;
+        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class);
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         String requestCallName = managed ? "requestGlobusTransferPaths" : "requestGlobusReferencePaths";
         allowedApiCalls.add(
                 Json.createObjectBuilder().add(URLTokenUtil.NAME, requestCallName).add(URLTokenUtil.HTTP_METHOD, "POST")
                         .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/requestGlobusUploadPaths")
                         .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        if(managed) {
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addGlobusFiles")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
                 .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        } else {
+            allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "addFiles")
+                    .add(URLTokenUtil.HTTP_METHOD, "POST")
+                    .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addFiles")
+                    .add(URLTokenUtil.TIMEOUT, timeoutSeconds));
+        }
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata")
                 .add(URLTokenUtil.HTTP_METHOD, "GET")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
-                .add(URLTokenUtil.TIMEOUT, 300));
+                .add(URLTokenUtil.TIMEOUT, 5));
         allowedApiCalls.add(
                 Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
                         .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
-                        .add(URLTokenUtil.TIMEOUT, 300));
+                        .add(URLTokenUtil.TIMEOUT, 5));
 
         return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
     }
@@ -3783,7 +3790,7 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
             params.add("endpoint", transferEndpoint);
         }
         params.add("files", files);
-        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) * 60;
+        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class);
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "monitorGlobusDownload")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
@@ -3797,11 +3804,11 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getDatasetMetadata")
                 .add(URLTokenUtil.HTTP_METHOD, "GET")
                 .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}")
-                .add(URLTokenUtil.TIMEOUT, 300));
+                .add(URLTokenUtil.TIMEOUT, 5));
         allowedApiCalls.add(
                 Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
                         .add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
-                        .add(URLTokenUtil.TIMEOUT, 300));
+                        .add(URLTokenUtil.TIMEOUT, 5));
 
         return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 37ec1afd4ed..b2ebea14835 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -569,7 +569,7 @@ static class MakeRequestResponse {
     // Single cache of open rules/permission requests
     private final Cache<String, JsonObject> downloadCache = Caffeine.newBuilder()
             .expireAfterWrite(
-                    Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) + 300, ChronoUnit.MINUTES))
+                    Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) + 5, ChronoUnit.MINUTES))
             .scheduler(Scheduler.systemScheduler()).evictionListener((downloadId, datasetId, cause) -> {
                 // Delete downloads that expire
                 logger.info("Download for " + downloadId + " expired");

From 9e8081523e28b21bff400369f70ec97d861d42fb Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sat, 25 Nov 2023 14:56:58 -0500
Subject: [PATCH 785/922] another iteration of the flyway script for
 calculating the initial sizes of all dvobjects. (#8549)

---
 .../V6.0.0.3__8549-collection-quotas.sql      | 53 +++++++++++--------
 1 file changed, 30 insertions(+), 23 deletions(-)

diff --git a/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql b/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
index f74d9bebe30..5644d06a682 100644
--- a/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
+++ b/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
@@ -1,9 +1,5 @@
--- Storage size column added:
-ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS storagesize BIGINT;
-
--- (work in progress! the table structure may change/the column may be moved out into
--- its own table. but the mechanics of the recursion are working)
-
+-- Adding the storagesize column to the datafile table:
+ALTER TABLE datafile ADD COLUMN IF NOT EXISTS storagesize BIGINT;
 -- The somewhat convoluted queries below populate the storage sizes for the entire
 -- DvObject tree, fast. It IS possible, to do it all with one recursive PostgresQL
 -- query, that will crawl the tree from the leaves (DataFiles) up and add up the
@@ -16,55 +12,66 @@ ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS storagesize BIGINT;
 -- have sub-collections. To take any sub-collections into account we are then running
 -- a recursive query - but then we only need to run it on the tree of Collections only,
 -- which should make it manageably fast on any real life instance. 
-
-UPDATE dvobject SET storagesize=0;
+----UPDATE dvobject SET storagesize=0;
 -- For datafiles, the storage size = main file size by default:
 -- (we are excluding any harvested files)
-UPDATE dvobject SET storagesize=COALESCE(f.filesize,0) FROM datafile f, dataset d WHERE f.id = dvobject.id AND dvobject.owner_id = d.id AND d.harvestingclient_id IS null;
+UPDATE datafile SET storagesize=COALESCE(filesize,0) FROM dataset d, dvobject o WHERE datafile.id = o.id AND o.owner_id = d.id AND d.harvestingclient_id IS null;
 -- ... but for ingested tabular files the size of the saved original needs to be added, since
 -- those also take space:
 -- (should be safe to assume that there are no *harvested ingested* files)
-UPDATE dvobject SET storagesize=dvobject.storagesize + COALESCE(datatable.originalFileSize,0) FROM datatable WHERE datatable.datafile_id = dvobject.id;
+UPDATE datafile SET storagesize=datafile.storagesize + COALESCE(datatable.originalFileSize,0) FROM datatable WHERE datatable.datafile_id = datafile.id;
+-- *Temporarily* add this "storagesize" column to the DvObject table.
+-- It will be used to calculate the storage sizes for all 3 types of DvObjects;
+-- a recursive query is used for this purpose further down, and it is convenient
+-- to use a column directly in the same table. 
+-- Once calculated, the values will will be moved to the permanent locations
+-- in the DataFile and StorageUse tables. 
+ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS tempStorageSize BIGINT;
 -- Now we can calculate storage sizes of each individual dataset (a simple sum
 -- of the storage sizes of all the files in the dataset):
 -- (excluding the harvested datasets; this is less important, since there should be
 -- significantly fewer datasets than files, but might as well)
-UPDATE dvobject SET storagesize=o.combinedStorageSize
-FROM (SELECT datasetobject.id, SUM(fileobject.storagesize) AS combinedStorageSize
-FROM dvobject fileobject, dvobject datasetobject
+UPDATE dvobject SET tempStorageSize=o.combinedStorageSize
+FROM (SELECT datasetobject.id, SUM(file.storagesize) AS combinedStorageSize
+FROM dvobject fileobject, dvobject datasetobject, datafile file
 WHERE fileobject.owner_id = datasetobject.id
+AND fileobject.id = file.id
 GROUP BY datasetobject.id) o, dataset ds WHERE o.id = dvobject.id AND dvobject.dtype='Dataset' AND dvobject.id = ds.id AND ds.harvestingclient_id IS null;
 -- ... and then we can repeat the same for collections, by setting the storage size
 -- to the sum of the storage sizes of the datasets *directly* in each collection:
 -- (no attemp is made yet to recursively count the sizes all the chilld sub-collections)
-UPDATE dvobject SET storagesize=o.combinedStorageSize
-FROM (SELECT collectionobject.id, SUM(datasetobject.storagesize) AS combinedStorageSize
+UPDATE dvobject SET tempStorageSize=o.combinedStorageSize
+FROM (SELECT collectionobject.id, SUM(datasetobject.tempStorageSize) AS combinedStorageSize
 FROM dvobject datasetobject, dvobject collectionobject
 WHERE datasetobject.owner_id = collectionobject.id
-AND datasetobject.storagesize IS NOT null 
+AND datasetobject.tempStorageSize IS NOT null 
 GROUP BY collectionobject.id) o WHERE o.id = dvobject.id AND dvobject.dtype='Dataverse';
 
 -- And now we will update the storage sizes of all the Collection ("Dataverse") objects
 -- that contain sub-collections, *recursively*, to add their sizes to the totals:
-WITH RECURSIVE treestorage (id, owner_id, storagesize, dtype) AS
+WITH RECURSIVE treestorage (id, owner_id, tempStorageSize, dtype) AS
 (
     -- All dataverses:
-    SELECT id, owner_id, storagesize, dtype
+    SELECT id, owner_id, tempStorageSize, dtype
     FROM dvobject
     WHERE dtype = 'Dataverse'
 
     UNION
 
     -- Recursive Member:
-    SELECT dvobject.id, treestorage.owner_id, dvobject.storagesize, treestorage.dtype
+    SELECT dvobject.id, treestorage.owner_id, dvobject.tempStorageSize, treestorage.dtype
     FROM treestorage, dvobject
     WHERE treestorage.id = dvobject.owner_id
     AND dvobject.dtype = 'Dataverse'
 )
-
-UPDATE dvobject SET storagesize=storagesize+(SELECT COALESCE(SUM(storagesize),0)
+UPDATE dvobject SET tempStorageSize=tempStorageSize+(SELECT COALESCE(SUM(tempStorageSize),0)
 FROM treestorage WHERE owner_id=dvobject.id)
---FROM treestorage ts
---WHERE ts.owner_id=dvobject.id
 WHERE dvobject.dtype = 'Dataverse'
 AND dvobject.id IN (SELECT owner_id FROM treestorage WHERE owner_id IS NOT null);
+
+-- And, finally, we can move these calculated storage sizes of datasets and
+-- collection to the dedicated new table StorageUse:
+INSERT INTO storageuse (dvobjectcontainer_id,sizeinbytes) (SELECT id, tempstoragesize FROM dvobject WHERE dtype = 'Dataverse');
+INSERT INTO storageuse (dvobjectcontainer_id,sizeinbytes) (SELECT d.id, o.tempstoragesize FROM dvobject o, dataset d WHERE o.id = d.id AND d.harvestingclient_id IS NULL);
+-- ... and drop the temporary column we added to DvObject earlier:
+ALTER TABLE dvobject DROP column tempStorageSize

From 1b95665e37f18871c35e48b36a32191975e024af Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sat, 25 Nov 2023 19:22:11 -0500
Subject: [PATCH 786/922] new object that defines the size limits for the
 current upload session, replacing the test class used during the proof of
 concept stage. (#8549)

---
 .../storageuse/UploadSessionQuotaLimit.java   | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java
new file mode 100644
index 00000000000..06bbe986f70
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java
@@ -0,0 +1,42 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.storageuse;
+
+/**
+ *
+ * @author landreev
+ */
+public class UploadSessionQuotaLimit {
+        private Long totalAllocatedInBytes = 0L; 
+        private Long totalUsageInBytes = 0L;
+        
+        public UploadSessionQuotaLimit(Long allocated, Long used) {
+            this.totalAllocatedInBytes = allocated;
+            this.totalUsageInBytes = used; 
+        }
+        
+        public Long getTotalAllocatedInBytes() {
+            return totalAllocatedInBytes;
+        }
+        
+        public void setTotalAllocatedInBytes(Long totalAllocatedInBytes) {
+            this.totalAllocatedInBytes = totalAllocatedInBytes;
+        }
+        
+        public Long getTotalUsageInBytes() {
+            return totalUsageInBytes;
+        }
+        
+        public void setTotalUsageInBytes(Long totalUsageInBytes) {
+            this.totalUsageInBytes = totalUsageInBytes;
+        }
+        
+        public Long getRemainingQuotaInBytes() {
+            if (totalUsageInBytes > totalAllocatedInBytes) {
+                return 0L; 
+            }
+            return totalAllocatedInBytes - totalUsageInBytes;
+        }
+    }
\ No newline at end of file

From 627a7b1974084a02922b6c63b5474a8540235d5e Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sat, 25 Nov 2023 20:22:40 -0500
Subject: [PATCH 787/922] a stub for the release note (#8549)

---
 doc/release-notes/8549-collection-quotas.md | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 doc/release-notes/8549-collection-quotas.md

diff --git a/doc/release-notes/8549-collection-quotas.md b/doc/release-notes/8549-collection-quotas.md
new file mode 100644
index 00000000000..8af222f09e0
--- /dev/null
+++ b/doc/release-notes/8549-collection-quotas.md
@@ -0,0 +1,2 @@
+This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
+

From a5196df759522aa2e30ee7ed3b52b24bfd84c301 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 26 Nov 2023 18:06:34 -0500
Subject: [PATCH 788/922] All the bits and pieces that haven't been committed
 yet. (#8549)

---
 doc/release-notes/8549-collection-quotas.md   |  1 +
 doc/sphinx-guides/source/api/native-api.rst   | 25 +++++
 .../edu/harvard/iq/dataverse/DataFile.java    | 17 ++++
 .../iq/dataverse/DataFileServiceBean.java     | 93 ++++++++-----------
 .../edu/harvard/iq/dataverse/Dataset.java     | 10 ++
 .../edu/harvard/iq/dataverse/Dataverse.java   |  7 +-
 .../edu/harvard/iq/dataverse/DvObject.java    | 25 +++--
 .../iq/dataverse/EditDatafilesPage.java       | 34 ++++---
 .../iq/dataverse/EjbDataverseEngine.java      | 10 ++
 .../harvard/iq/dataverse/api/Dataverses.java  | 18 ++++
 .../datadeposit/MediaResourceManagerImpl.java |  6 +-
 .../datasetutility/AddReplaceFileHelper.java  |  5 +-
 .../engine/command/CommandContext.java        |  3 +
 .../impl/CreateNewDataFilesCommand.java       | 31 ++++---
 .../command/impl/DeleteDataFileCommand.java   | 14 +++
 .../impl/GetCollectionQuotaCommand.java       | 16 +++-
 .../impl/GetCollectionStorageUseCommand.java  | 49 ++++++++++
 .../command/impl/UningestFileCommand.java     | 13 +++
 .../dataverse/ingest/IngestServiceBean.java   | 47 ++++++++--
 .../iq/dataverse/storageuse/StorageQuota.java |  4 +-
 .../iq/dataverse/storageuse/StorageUse.java   | 10 +-
 .../storageuse/StorageUseServiceBean.java     | 45 +++++++--
 src/main/java/propertyFiles/Bundle.properties |  3 +-
 .../V6.0.0.3__8549-collection-quotas.sql      | 63 ++++++-------
 .../dataverse/engine/TestCommandContext.java  |  6 ++
 25 files changed, 399 insertions(+), 156 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java

diff --git a/doc/release-notes/8549-collection-quotas.md b/doc/release-notes/8549-collection-quotas.md
index 8af222f09e0..e3bf37425ea 100644
--- a/doc/release-notes/8549-collection-quotas.md
+++ b/doc/release-notes/8549-collection-quotas.md
@@ -1,2 +1,3 @@
 This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
+Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the deployments. 
 
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 7ae1194ebc6..1b56ee132e8 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -754,6 +754,31 @@ The following attributes are supported:
 * ``affiliation`` Affiliation
 * ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
 
+.. _collection-storage-quotas:
+  
+Collection Storage Quotas
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: 
+
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota"
+
+Will output the storage quota allocated (in bytes), or a message indicating that the quota is not defined for the collection.
+
+To set or change the storage allocation quota for a collection:
+
+.. code-block:: 
+
+  curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota/$SIZE_IN_BYTES"
+
+To delete a storage quota configured for a collection:
+
+.. code-block:: 
+
+  curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota"
+
+
+
 
 Datasets
 --------
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index 407282a5372..2770118d41b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -640,6 +640,23 @@ public String getFriendlySize() {
             return BundleUtil.getStringFromBundle("file.sizeNotAvailable");
         }
     }
+    
+    /**
+     * Experimental - record the pre-calculated "storage size" of the file, and 
+     * all its associated auxiliary file objects:
+
+    @Column(nullable = true)
+    private Long storageSize;
+    
+
+    public Long getStorageSize() {
+        return storageSize; 
+    }
+    
+    public void setStorageSize(Long storageSize) {
+        this.storageSize = storageSize; 
+    }
+    * */
 
     public boolean isRestricted() {
         return restricted;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 332a39912d2..42c47505b70 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -8,6 +8,9 @@
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import edu.harvard.iq.dataverse.storageuse.StorageUseServiceBean;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
 import edu.harvard.iq.dataverse.util.FileSortFieldAndOrder;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -41,8 +44,6 @@
  *
  * @author Leonid Andreev
  * 
- * Basic skeleton of the new DataFile service for DVN 4.0
- * 
  */
 
 @Stateless
@@ -66,6 +67,9 @@ public class DataFileServiceBean implements java.io.Serializable {
     
     @EJB SystemConfig systemConfig;
     
+    @EJB
+    StorageUseServiceBean storageUseService; 
+    
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
     
@@ -139,39 +143,6 @@ public class DataFileServiceBean implements java.io.Serializable {
      */
     public static final String MIME_TYPE_PACKAGE_FILE = "application/vnd.dataverse.file-package";
     
-    public class UserStorageQuota {
-        private Long totalAllocatedInBytes = 0L; 
-        private Long totalUsageInBytes = 0L;
-        
-        public UserStorageQuota(Long allocated, Long used) {
-            this.totalAllocatedInBytes = allocated;
-            this.totalUsageInBytes = used; 
-        }
-        
-        public Long getTotalAllocatedInBytes() {
-            return totalAllocatedInBytes;
-        }
-        
-        public void setTotalAllocatedInBytes(Long totalAllocatedInBytes) {
-            this.totalAllocatedInBytes = totalAllocatedInBytes;
-        }
-        
-        public Long getTotalUsageInBytes() {
-            return totalUsageInBytes;
-        }
-        
-        public void setTotalUsageInBytes(Long totalUsageInBytes) {
-            this.totalUsageInBytes = totalUsageInBytes;
-        }
-        
-        public Long getRemainingQuotaInBytes() {
-            if (totalUsageInBytes > totalAllocatedInBytes) {
-                return 0L; 
-            }
-            return totalAllocatedInBytes - totalUsageInBytes;
-        }
-    }
-    
     public DataFile find(Object pk) {
         return em.find(DataFile.class, pk);
     }   
@@ -1396,28 +1367,38 @@ public Embargo findEmbargo(Long id) {
         return d.getEmbargo();
     }
     
-    public Long getStorageUsageByCreator(AuthenticatedUser user) {
-        Query query = em.createQuery("SELECT SUM(o.filesize) FROM DataFile o WHERE o.creator.id=:creatorId");
-        
-        try {
-            Long totalSize = (Long)query.setParameter("creatorId", user.getId()).getSingleResult();
-            logger.info("total size for user: "+totalSize);
-            return totalSize == null ? 0L : totalSize; 
-        } catch (NoResultException nre) { // ?
-            logger.info("NoResultException, returning 0L");
-            return 0L;
+    /**
+     * Checks if the supplied DvObjectContainer (Dataset or Collection; although
+     * only collection-level storage quotas are officially supported as if now)
+     * has a quota configured, and if not, keeps checking if any of the direct
+     * ancestor Collections further up have a configured quota. If it finds one, 
+     * it will retrieve the current total content size for that specific ancestor 
+     * dvObjectContainer and use it to define the quota limit for the upload
+     * session in progress. 
+     * 
+     * @param parent - DvObjectContainer, Dataset or Collection
+     * @return upload session size limit spec, or null if quota not defined on 
+     * any of the ancestor DvObjectContainers
+     */
+    public UploadSessionQuotaLimit getUploadSessionQuotaLimit(DvObjectContainer parent) {
+        DvObjectContainer testDvContainer = parent; 
+        StorageQuota quota = testDvContainer.getStorageQuota();
+        while (quota == null && testDvContainer.getOwner() != null) {
+            testDvContainer = testDvContainer.getOwner();
+            quota = testDvContainer.getStorageQuota();
+            if (quota != null) {
+                break;
+            }
+        }    
+        if (quota == null || quota.getAllocation() == null) {
+            return null; 
         }
-    }
-    
-    public UserStorageQuota getUserStorageQuota(AuthenticatedUser user, Dataset dataset) {
-        // this is for testing only - one pre-set, installation-wide quota limit
-        // for everybody:
-        Long totalAllocated = systemConfig.getTestStorageQuotaLimit();
-        // again, this is for testing only - we are only counting the total size
-        // of all the files created by this user; it will likely be a much more 
-        // complex calculation in real life applications:
-        Long totalUsed = getStorageUsageByCreator(user); 
         
-        return new UserStorageQuota(totalAllocated, totalUsed);
+        // Note that we are checking the recorded storage use not on the 
+        // immediate parent necessarily, but on the specific ancestor 
+        // DvObjectContainer on which the storage quota is defined:
+        Long currentSize = storageUseService.findStorageSizeByDvContainerId(testDvContainer.getId()); 
+        
+        return new UploadSessionQuotaLimit(quota.getAllocation(), currentSize);
     }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataset.java b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
index 245bdf0efd2..a2f560bc959 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataset.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataset.java
@@ -35,6 +35,7 @@
 import jakarta.persistence.TemporalType;
 
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.storageuse.StorageUse;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 
@@ -189,6 +190,10 @@ public void setTemplate(Template template) {
     }
 
     public Dataset() {
+        this(false);
+    }
+    
+    public Dataset(boolean isHarvested) {
         DatasetVersion datasetVersion = new DatasetVersion();
         datasetVersion.setDataset(this);
         datasetVersion.setVersionState(DatasetVersion.VersionState.DRAFT);
@@ -196,6 +201,11 @@ public Dataset() {
         datasetVersion.setVersionNumber((long) 1);
         datasetVersion.setMinorVersionNumber((long) 0);
         versions.add(datasetVersion);
+        
+        if (!isHarvested) {
+            StorageUse storageUse = new StorageUse(this); 
+            this.setStorageUse(storageUse);
+        }
     }
     
     /**
diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
index 682c1dc6744..c1de9d63410 100644
--- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java
@@ -3,6 +3,7 @@
 import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearch;
+import edu.harvard.iq.dataverse.storageuse.StorageUse;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 
@@ -103,7 +104,11 @@ public enum DataverseType {
      * dataverses.
      */
     protected boolean permissionRoot;
-
+    
+    public Dataverse() {
+        StorageUse storageUse = new StorageUse(this); 
+        this.setStorageUse(storageUse);
+    }
     
     public DataverseType getDataverseType() {
         return dataverseType;
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
index b86fabd0a07..515d9f9f153 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
@@ -157,10 +157,7 @@ public String visit(DataFile df) {
     private String identifier;
     
     private boolean identifierRegistered;
-    
-    @Column(nullable = true)
-    private Long storageSize; 
-    
+        
     private transient GlobalId globalId = null;
     
     @OneToMany(mappedBy = "dvObject", cascade = CascadeType.ALL, orphanRemoval = true)
@@ -185,7 +182,7 @@ public void setAlternativePersistentIndentifiers(Set<AlternativePersistentIdenti
     @OneToOne(mappedBy = "definitionPoint",cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST}, orphanRemoval=true)
     private StorageQuota storageQuota;
     
-    @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE}, orphanRemoval=true)
+    @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE, CascadeType.PERSIST}, orphanRemoval=true)
     private StorageUse storageUse;
     
     
@@ -470,14 +467,6 @@ public void setStorageIdentifier(String storageIdentifier) {
         this.storageIdentifier = storageIdentifier;
     }
     
-    public Long getStorageSize() {
-        return storageSize; 
-    }
-    
-    public void setStorageSize(Long storageSize) {
-        this.storageSize = storageSize; 
-    }
-    
     public StorageQuota getStorageQuota() {
         return storageQuota;
     }
@@ -495,4 +484,14 @@ public void setStorageQuota(StorageQuota storageQuota) {
     
     @OneToMany(mappedBy = "definitionPoint",cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST}, orphanRemoval=true)
     List<RoleAssignment> roleAssignments;
+    
+    /**
+     * Should only be used in constructors for DvObjectContainers (Datasets and 
+     * Collections), to make sure new entries are created and persisted in the 
+     * database StorageUse table for every DvObject container we create.
+     * @param storageUse 
+     */
+    public void setStorageUse(StorageUse storageUse) {
+        this.storageUse = storageUse;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index c40399e160f..a3256c9159f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -2,7 +2,6 @@
 
 import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean;
 import edu.harvard.iq.dataverse.DataFile.ChecksumType;
-import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.api.AbstractApiBean;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.Permission;
@@ -38,6 +37,7 @@
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.Setting;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -206,7 +206,7 @@ public enum Referrer {
     private final int NUMBER_OF_SCROLL_ROWS = 25;
 
     private DataFile singleFile = null;
-    private UserStorageQuota userStorageQuota = null; 
+    private UploadSessionQuotaLimit uploadSessionQuota = null; 
 
     public DataFile getSingleFile() {
         return singleFile;
@@ -359,7 +359,7 @@ public String getHumanMaxTotalUploadSizeInBytes() {
     }
     
     public boolean isStorageQuotaEnforced() {
-        return userStorageQuota != null; 
+        return uploadSessionQuota != null; 
     }
 
     public Long getMaxIngestSizeInBytes() {
@@ -530,8 +530,10 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo
 
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
         if (systemConfig.isStorageQuotasEnforced()) {
-            this.userStorageQuota = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset);
-            this.maxTotalUploadSizeInBytes = userStorageQuota.getRemainingQuotaInBytes();
+            this.uploadSessionQuota = datafileService.getUploadSessionQuotaLimit(dataset);
+            if (this.uploadSessionQuota != null) {
+                this.maxTotalUploadSizeInBytes = uploadSessionQuota.getRemainingQuotaInBytes();
+            }
         } else {
             this.maxTotalUploadSizeInBytes = null; 
         }
@@ -547,7 +549,7 @@ public String initCreateMode(String modeToken, DatasetVersion version, MutableBo
     }
     
     public boolean isQuotaExceeded() {
-        return systemConfig.isStorageQuotasEnforced() && userStorageQuota != null && userStorageQuota.getRemainingQuotaInBytes() == 0;
+        return systemConfig.isStorageQuotasEnforced() && uploadSessionQuota != null && uploadSessionQuota.getRemainingQuotaInBytes() == 0;
     }
 
     public String init() {
@@ -592,8 +594,10 @@ public String init() {
         clone = workingVersion.cloneDatasetVersion();
         this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getEffectiveStorageDriverId());
         if (systemConfig.isStorageQuotasEnforced()) {
-            this.userStorageQuota = datafileService.getUserStorageQuota((AuthenticatedUser) session.getUser(), dataset);
-            this.maxTotalUploadSizeInBytes = userStorageQuota.getRemainingQuotaInBytes();
+            this.uploadSessionQuota = datafileService.getUploadSessionQuotaLimit(dataset);
+            if (this.uploadSessionQuota != null) {
+                this.maxTotalUploadSizeInBytes = uploadSessionQuota.getRemainingQuotaInBytes();
+            }
         }
         this.maxIngestSizeInBytes = systemConfig.getTabularIngestSizeLimit();
         this.humanPerFormatTabularLimits = populateHumanPerFormatTabularLimits();
@@ -1098,7 +1102,11 @@ public String save() {
             }
 
             // Try to save the NEW files permanently: 
-            List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true);
+            // ... but first, refresh the session quota specifiction, if defined:
+            if (systemConfig.isStorageQuotasEnforced()) {
+                this.uploadSessionQuota = datafileService.getUploadSessionQuotaLimit(dataset);
+            }
+            List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true, uploadSessionQuota);
             
             // reset the working list of fileMetadatas, as to only include the ones
             // that have been added to the version successfully: 
@@ -1529,7 +1537,7 @@ public void handleDropBoxUpload(ActionEvent event) {
                 // zip file.
                 //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream");
                 //CreateDataFileResult createDataFilesResult = FileUtil.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream", null, null, systemConfig);
-                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, dropBoxStream, fileName, "application/octet-stream", null, userStorageQuota, null);
+                Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, dropBoxStream, fileName, "application/octet-stream", null, uploadSessionQuota, null);
                 CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                 datafiles = createDataFilesResult.getDataFiles();
                 Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
@@ -2068,9 +2076,9 @@ public void handleFileUpload(FileUploadEvent event) throws IOException {
                 // dataset that does not yet exist in the database. We must 
                 // use the version of the Create New Files constructor that takes
                 // the parent Dataverse as the extra argument:
-                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null, null, null, workingVersion.getDataset().getOwner());
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, uploadSessionQuota, null, null, null, workingVersion.getDataset().getOwner());
             } else {
-                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, userStorageQuota, null);
+                cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, uFile.getInputStream(), uFile.getFileName(), uFile.getContentType(), null, uploadSessionQuota, null);
             }
             CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
 
@@ -2208,7 +2216,7 @@ public void handleExternalUpload() {
                 
                 try {
   
-                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, userStorageQuota, checksumValue, checksumType, fileSize, parent);
+                    Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequestService.getDataverseRequest(), workingVersion, null, fileName, contentType, fullStorageIdentifier, uploadSessionQuota, checksumValue, checksumType, fileSize, parent);
                     CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
                     datafiles = createDataFilesResult.getDataFiles();
                     Optional.ofNullable(editDataFilesPageHelper.getHtmlErrorMessage(createDataFilesResult)).ifPresent(errorMessage -> errorMessages.add(errorMessage));
diff --git a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java
index bad8903c091..5a689c06019 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EjbDataverseEngine.java
@@ -31,6 +31,7 @@
 import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.storageuse.StorageUseServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -184,6 +185,9 @@ public class EjbDataverseEngine {
     @EJB
     ConfirmEmailServiceBean confirmEmailService;
     
+    @EJB
+    StorageUseServiceBean storageUseService; 
+    
     @EJB
     EjbDataverseEngineInner innerEngine;
     
@@ -528,6 +532,12 @@ public DataverseLinkingServiceBean dvLinking() {
                 public DatasetLinkingServiceBean dsLinking() {
                     return dsLinking;
                 }
+                
+                @Override
+                public StorageUseServiceBean storageUse() {
+                    return storageUseService;
+                }
+                
                 @Override
                 public DataverseEngine engine() {
                     return new DataverseEngine() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 6539e42b07d..b1e7559f858 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -46,6 +46,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseLinkingDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.DeleteExplicitGroupCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionQuotaCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionStorageUseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetRootCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseCommand;
 import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseStorageSizeCommand;
@@ -979,6 +980,23 @@ public Response deleteCollectionQuota(@Context ContainerRequestContext crc, @Pat
         }
     }
     
+    /**
+     *
+     * @param crc
+     * @param identifier
+     * @return
+     * @throws edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse 
+     * @todo: add an optional parameter that would force the recorded storage use
+     * to be recalculated (or should that be a POST version of this API?)
+     */
+    @GET
+    @AuthRequired
+    @Path("{identifier}/storage/use")
+    public Response getCollectionStorageUse(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier) throws WrappedResponse {
+        return response(req -> ok(MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.storage.use"),
+                execCommand(new GetCollectionStorageUseCommand(req, findDataverseOrDie(identifier))))), getRequestUser(crc));
+    }
+
     @GET
     @AuthRequired
     @Path("{identifier}/roles")
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
index 15838a09456..fcbb1315c24 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java
@@ -11,7 +11,6 @@
 import edu.harvard.iq.dataverse.PermissionServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
-import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
@@ -19,6 +18,7 @@
 import edu.harvard.iq.dataverse.engine.command.impl.UpdateDatasetVersionCommand;
 import edu.harvard.iq.dataverse.ingest.IngestServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.ConstraintViolationUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
@@ -307,9 +307,9 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au
 
             try {
                 //CreateDataFileResult createDataFilesResponse =  FileUtil.createDataFiles(editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, null, systemConfig);
-                UserStorageQuota quota = null; 
+                UploadSessionQuotaLimit quota = null; 
                 if (systemConfig.isStorageQuotasEnforced()) {
-                    quota = dataFileService.getUserStorageQuota(user, dataset);
+                    quota = dataFileService.getUploadSessionQuotaLimit(dataset);
                 }
                 Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvReq, editVersion, deposit.getInputStream(), uploadedZipFilename, guessContentTypeForMe, null, quota, null);
                 CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
index d44388f39f7..0143fced87c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -61,6 +61,7 @@
 import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder;
 import org.apache.commons.io.IOUtils;
 import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDataFilesCommand;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
 import static jakarta.ws.rs.core.Response.Status.BAD_REQUEST;
 
 /**
@@ -1212,9 +1213,9 @@ private boolean step_030_createNewFilesViaIngest(){
                     this.newCheckSumType,
                     this.systemConfig);*/
             
-            DataFileServiceBean.UserStorageQuota quota = null; 
+            UploadSessionQuotaLimit quota = null; 
             if (systemConfig.isStorageQuotasEnforced()) {
-                quota = fileService.getUserStorageQuota(dvRequest.getAuthenticatedUser(), dataset);
+                quota = fileService.getUploadSessionQuotaLimit(dataset);
             }
             Command<CreateDataFileResult> cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType);
             CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java
index 55a375acb6c..f74c1222bb0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/CommandContext.java
@@ -39,6 +39,7 @@
 import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.storageuse.StorageUseServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.workflow.WorkflowServiceBean;
 import java.util.Stack;
@@ -126,6 +127,8 @@ public interface CommandContext {
     public UserNotificationServiceBean notifications();
 
     public AuthenticationServiceBean authentication();
+    
+    public StorageUseServiceBean storageUse();
 
     public SystemConfig systemConfig();
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 7a32e398b8f..3a21345448b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -13,8 +13,8 @@
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
-import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -74,7 +74,7 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private final InputStream inputStream;
     private final String fileName;
     private final String suppliedContentType; 
-    private final UserStorageQuota quota;
+    private final UploadSessionQuotaLimit quota;
     // parent Dataverse must be specified when the command is called on Create 
     // of a new dataset that does not exist in the database yet (for the purposes
     // of authorization - see getRequiredPermissions() below):
@@ -85,18 +85,18 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private DataFile.ChecksumType newCheckSumType;
     private final Long newFileSize;
 
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum) {
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum) {
         this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, null);
     }
     
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType) {
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum, DataFile.ChecksumType newCheckSumType) {
         this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, null, null);
     }
     
     // This version of the command must be used when files are created in the 
     // context of creating a brand new dataset (from the Add Dataset page):
     
-    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize, Dataverse dataverse) {
+    public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize, Dataverse dataverse) {
         super(aRequest, dataverse);
         
         this.version = version;
@@ -701,7 +701,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
         
         DataFile datafile = FileUtil.createSingleDataFile(version, newFile, newStorageIdentifier, fileName, finalType, newCheckSumType, newCheckSum);
 
-        if (datafile != null && ((newFile != null) || (newStorageIdentifier != null))) {
+        if (datafile != null) {
 
             if (warningMessage != null) {
                 createIngestFailureReport(datafile, warningMessage);
@@ -712,14 +712,19 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
             }
             datafiles.add(datafile);
 
-            // Update quota (not necessary in the context of direct upload, will be done later)
-            // On a second thought, @todo: we should delay updating the storage size/quotas
-            // until the file is saved and finalized, for all upload cases!
-            if (newFile != null) {
-                if (fileSize > 0 && quota != null) {
-                    quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
-                }
+            // Update the quota definition for the *current upload session*
+            // This is relevant for the uploads going through the UI page 
+            // (where there may be an appreciable amount of time between the user
+            // uploading the files and clicking "save". The file size should be 
+            // available here for both direct and local uploads via the UI. 
+            // It is not yet available if this is direct-via-API - but 
+            // for API uploads the quota check will be enforced during the final 
+            // save. 
+            if (fileSize > 0 && quota != null) {
+                logger.info("Setting total usage in bytes to " + (quota.getTotalUsageInBytes() + fileSize));
+                quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + fileSize);
             }
+
             return CreateDataFileResult.success(fileName, finalType, datafiles);
         }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java
index 83d0f877d61..e2730ec06d3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteDataFileCommand.java
@@ -235,6 +235,20 @@ public String describe() {
 
     @Override
     public boolean onSuccess(CommandContext ctxt, Object r) {
+        // Adjust the storage use for the parent containers: 
+        if (!doomed.isHarvested()) {
+            long storedSize = doomed.getFilesize();
+            // ingested tabular data files also have saved originals that 
+            // are counted as "storage use"
+            Long savedOriginalSize = doomed.getOriginalFileSize(); 
+            if (savedOriginalSize != null) {
+                // Note that DataFile.getFilesize() can return -1 (for "unknown"):
+                storedSize = storedSize > 0 ? storedSize + savedOriginalSize : savedOriginalSize; 
+            }
+            if (storedSize > 0) {
+                ctxt.storageUse().incrementStorageSizeRecursively(doomed.getOwner().getId(), (0L - storedSize));
+            }
+        }
         /**
          * We *could* re-index the entire dataset but it's more efficient to
          * target individual files for deletion, which should always be drafts.
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java
index f07fde9508e..49f14e7c280 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionQuotaCommand.java
@@ -1,6 +1,5 @@
 package edu.harvard.iq.dataverse.engine.command.impl;
 
-import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
@@ -8,9 +7,9 @@
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
-import edu.harvard.iq.dataverse.util.BundleUtil;
-import java.io.IOException;
-import java.util.List;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
 import java.util.logging.Logger;
 
 /**
@@ -19,7 +18,7 @@
  * The command doesn't do much. It's sole purpose is to check the permissions
  * when it's called by the /api/dataverses/.../storage/quota api. 
  */
-@RequiredPermissions(Permission.ManageDataversePermissions)
+// @RequiredPermissions - none defined, dynamic
 public class GetCollectionQuotaCommand  extends AbstractCommand<Long> {
 
     private static final Logger logger = Logger.getLogger(GetCollectionQuotaCommand.class.getCanonicalName());
@@ -39,6 +38,13 @@ public Long execute(CommandContext ctxt) throws CommandException {
         }
         
         return null;
+    }
+
+    @Override
+    public Map<String, Set<Permission>> getRequiredPermissions() {
+        return Collections.singletonMap("",
+                dataverse.isReleased() ? Collections.<Permission>emptySet()
+                : Collections.singleton(Permission.ViewUnpublishedDataverse));
     }    
 }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java
new file mode 100644
index 00000000000..40b3128b80d
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java
@@ -0,0 +1,49 @@
+/*
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
+ * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
+ */
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import java.util.logging.Logger;
+
+/**
+ *
+ * @author landreev
+ */
+@RequiredPermissions(Permission.ManageDataversePermissions)
+// alternatively, we could make it dynamic - public for published collections
+// and Permission.ViewUnpublishedDataverse required otherwise (?)
+public class GetCollectionStorageUseCommand extends AbstractCommand<Long> {
+
+    private static final Logger logger = Logger.getLogger(GetCollectionStorageUseCommand.class.getCanonicalName());
+    
+    private final Dataverse collection;
+    
+    public GetCollectionStorageUseCommand(DataverseRequest aRequest, Dataverse target) {
+        super(aRequest, target);
+        collection = target;
+    } 
+        
+    @Override
+    public Long execute(CommandContext ctxt) throws CommandException {
+               
+        if (collection == null) {
+            throw new CommandException("null collection passed to get storage use command", this);
+        }
+        return ctxt.storageUse().findStorageSizeByDvContainerId(collection.getId());        
+    }
+
+    /*@Override
+    public Map<String, Set<Permission>> getRequiredPermissions() {
+        return Collections.singletonMap("",
+                dataverse.isReleased() ? Collections.<Permission>emptySet()
+                : Collections.singleton(Permission.ViewUnpublishedDataverse));
+    }*/   
+}
\ No newline at end of file
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java
index f2b89746160..3e85630dd59 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UningestFileCommand.java
@@ -105,6 +105,7 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
         // all the attribute of the file that are stored in the database: 
         
         // the file size: 
+        long archivalFileSize = uningest.getFilesize();
         uningest.setFilesize(storedOriginalFileSize);
         
         // original file format:
@@ -170,8 +171,20 @@ protected void executeImpl(CommandContext ctxt) throws CommandException {
             logger.warning("Io Exception deleting all aux objects : " + uningest.getId());
         }
         
+        // Finally, adjust the recorded storage use for the ancestral 
+        // DvObjectContainers (the parent dataset + all the parent collections
+        // up to the root):
+        if (archivalFileSize > 0) {
+            ctxt.storageUse().incrementStorageSizeRecursively(uningest.getOwner().getId(), (0L - archivalFileSize));
+        }
+        
     }
     
+    @Override
+    public boolean onSuccess(CommandContext ctxt, Object r) {
+        
+        return true; 
+    }
     
     private void resetIngestStats(DataFile uningest, CommandContext ctxt){
         
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 391f28674e4..9ef0cf07808 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -30,7 +30,6 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DataFileCategory;
 import edu.harvard.iq.dataverse.DataFileServiceBean;
-import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.DataTable;
 import edu.harvard.iq.dataverse.DatasetField;
 import edu.harvard.iq.dataverse.DatasetFieldServiceBean;
@@ -73,6 +72,8 @@
 import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReader;
 import edu.harvard.iq.dataverse.ingest.tabulardata.impl.plugins.por.PORFileReaderSpi;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.storageuse.StorageUseServiceBean;
+import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
 import edu.harvard.iq.dataverse.util.*;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 
@@ -151,6 +152,8 @@ public class IngestServiceBean {
     @EJB
     AuxiliaryFileServiceBean auxiliaryFileService;
     @EJB
+    StorageUseServiceBean storageUseService; 
+    @EJB
     SystemConfig systemConfig;
 
     @Resource(lookup = "java:app/jms/queue/ingest")
@@ -185,7 +188,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
             List<DataFile> newFiles,
             DataFile fileToReplace,
             boolean tabIngest, 
-            UserStorageQuota quota) /*throws FileExceedsMaxSizeException, FileExceedsStorageQuotaException*/ {
+            UploadSessionQuotaLimit quota) /*throws FileExceedsMaxSizeException, FileExceedsStorageQuotaException*/ {
         List<DataFile> ret = new ArrayList<>();
 
         if (newFiles != null && newFiles.size() > 0) {
@@ -360,20 +363,41 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                     }
                 }
                 
-                // We will perform (another) quota check, and if still under quota
-                // (it's not impossible that somebody else may have uploaded more 
-                // stuff into the same collection/dataset etc., before this user 
-                // decided to click "save", for example!)
+                // If quotas are enforced, we will perform a quota check here. 
+                // If this is an upload via the UI, we must have already 
+                // performed this check once. But it is possible that somebody 
+                // else may have added more data to the same collection/dataset 
+                // etc., before this user was ready to click "save", so this is
+                // necessary. For other cases, such as the direct uploads via 
+                // the API, this is the single point in the workflow where  
+                // storage quotas are enforced. 
         
                 if (systemConfig.isStorageQuotasEnforced() && quota != null) {
                     long storageQuotaLimit = quota.getRemainingQuotaInBytes();
                     if (confirmedFileSize > storageQuotaLimit) {
                         savedSuccess = false; 
+                        logger.warning("file size over quota limit, skipping");
+                        // @todo: we need to figure out how to better communicate
+                        // this (potentially partial) failure to the user.  
                         //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(confirmedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                    } else {
+                        // Update storage use for all the parent dvobjects: 
+                        // @todo: Do we want to do this after after *each* file is saved? - there may be 
+                        // quite a few files being saved here all at once. We could alternatively
+                        // perform this update only once, after this loop is completed (are there any
+                        // risks/accuracy loss?)
+                        // This update is performed with a direct native query that 
+                        // is supposed to be quite fast. But still. 
+                        logger.info("Incrementing recorded storage use by "+confirmedFileSize+" bytes for dataset "+dataset.getId());
+                        // (@todo: need to consider what happens when this code is called on Create?)
+                        storageUseService.incrementStorageSizeRecursively(dataset.getId(), confirmedFileSize);
+                        // Adjust quota: 
+                        logger.info("Setting total usage in bytes to "+(quota.getTotalUsageInBytes() + confirmedFileSize));
+                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + confirmedFileSize);
                     }
                 }
 
-                logger.fine("Done! Finished saving new files in permanent storage and adding them to the dataset.");
+                logger.fine("Done! Finished saving new file in permanent storage and adding them to the dataset.");
                 boolean belowLimit = false;
 
                 try {
@@ -1078,7 +1102,14 @@ public boolean ingestAsTabular(Long datafile_id) {
                     }
                 }
 
-                if (!databaseSaveSuccessful) {
+                if (databaseSaveSuccessful) {
+                    // Add the size of the tab-delimited version of the data file 
+                    // that we have produced and stored to the recorded storage 
+                    // size of all the ancestor DvObjectContainers: 
+                    if (dataFile.getFilesize() > 0) {
+                        storageUseService.incrementStorageSizeRecursively(dataFile.getOwner().getId(), dataFile.getFilesize());
+                    }
+                } else {
                     logger.warning("Ingest failure (failed to save the tabular data in the database; file left intact as uploaded).");
                     return false;
                 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
index 68ff6d95d00..0cfebe4167a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
@@ -66,7 +66,9 @@ public void setId(Long id) {
     
     public StorageQuota() {}
     
-    /*public String getAssigneeIdentifier() {
+    /**
+     * Could be uncommented if/when we want to add per-user quotas (see above)
+    public String getAssigneeIdentifier() {
         return assigneeIdentifier;
     }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
index c91ff69b0ae..11a2a8b706c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
@@ -24,14 +24,14 @@
 @NamedQueries({
     @NamedQuery(name = "StorageUse.findByteSizeByDvContainerId",query = "SELECT su.sizeInBytes FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId "),
     @NamedQuery(name = "StorageUse.findByDvContainerId",query = "SELECT su FROM StorageUse su WHERE su.dvObjectContainer.id =:dvObjectId "),
-    @NamedQuery(name = "StorageUse.incrementByteSizeByDvContainerId", query = "UPDATE StorageUse SET sizeInBytes = (sizeInBytes + :fileSize) WHERE dvObjectContainer.id=dv.ObjectId")
+    @NamedQuery(name = "StorageUse.incrementByteSizeByDvContainerId", query = "UPDATE StorageUse su SET su.sizeInBytes = su.sizeInBytes +:fileSize WHERE su.dvObjectContainer.id =:dvObjectId")
 })
 @Entity
 public class StorageUse implements Serializable {
 
     private static final long serialVersionUID = 1L;
     @Id
-    @GeneratedValue(strategy = GenerationType.AUTO)
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
     private Long id;
 
     public Long getId() {
@@ -49,6 +49,12 @@ public void setId(Long id) {
     @Column
     private Long sizeInBytes = null; 
     
+    public StorageUse() {}
+    
+    public StorageUse(DvObjectContainer dvObjectContainer) {
+        this(dvObjectContainer, 0L);
+    }
+    
     public StorageUse(DvObjectContainer dvObjectContainer, Long sizeInBytes) {
         this.dvObjectContainer = dvObjectContainer;
         this.sizeInBytes = sizeInBytes;
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
index fd04344c234..e92ba43e950 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
@@ -4,11 +4,10 @@
  */
 package edu.harvard.iq.dataverse.storageuse;
 
-import edu.harvard.iq.dataverse.DataverseServiceBean;
-import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.DvObjectContainer;
-import jakarta.ejb.EJB;
 import jakarta.ejb.Stateless;
+import jakarta.ejb.TransactionAttribute;
+import jakarta.ejb.TransactionAttributeType;
 import jakarta.inject.Named;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
@@ -22,8 +21,6 @@
 @Named
 public class StorageUseServiceBean  implements java.io.Serializable {
     private static final Logger logger = Logger.getLogger(StorageUseServiceBean.class.getCanonicalName());
-    @EJB
-    DataverseServiceBean dataverseService;
     
     @PersistenceContext(unitName = "VDCNet-ejbPU")
     private EntityManager em;
@@ -32,8 +29,16 @@ public StorageUse findByDvContainerId(Long dvObjectId) {
         return em.createNamedQuery("StorageUse.findByDvContainerId", StorageUse.class).setParameter("dvObjectId", dvObjectId).getSingleResult();
     }
     
+    /**
+     * Looks up the current storage use size, using a named query in a new 
+     * transaction
+     * @param dvObjectId
+     * @return 
+     */
+    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
     public Long findStorageSizeByDvContainerId(Long dvObjectId) {
-        return em.createNamedQuery("StorageUse.findByteSizeByDvContainerId", Long.class).setParameter("dvObjectId", dvObjectId).getSingleResult();
+        Long res = em.createNamedQuery("StorageUse.findByteSizeByDvContainerId", Long.class).setParameter("dvObjectId", dvObjectId).getSingleResult();
+        return res == null ? 0L : res;
     }
     
     public void incrementStorageSizeHierarchy(DvObjectContainer dvObject, Long filesize) {
@@ -46,7 +51,6 @@ public void incrementStorageSizeHierarchy(DvObjectContainer dvObject, Long files
     }
     
     /**
-     * Should this be done in a new transaction?
      * @param dvObject
      * @param filesize 
      */
@@ -62,4 +66,31 @@ public void incrementStorageSize(DvObjectContainer dvObject, Long filesize) {
         }
     }
     
+    /**
+     * Increments the recorded storage size for all the dvobject parents of a
+     * datafile, recursively. 
+     * @param dvObjectContainerId database id of the immediate parent (dataset)
+     * @param increment size in bytes of the file(s) being added 
+     */
+    @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
+    public void incrementStorageSizeRecursively(Long dvObjectContainerId, Long increment) {
+        //@todo should throw exceptions if either parameter is null
+        String queryString = "WITH RECURSIVE uptree (id, owner_id) AS\n"
+                + "("
+                + "    SELECT id, owner_id\n"
+                + "    FROM dvobject\n"
+                + "    WHERE id=" + dvObjectContainerId + "\n"
+                + "    UNION ALL\n"
+                + "    SELECT dvobject.id, dvobject.owner_id\n"
+                + "    FROM dvobject\n"
+                + "    JOIN uptree ON dvobject.id = uptree.owner_id)\n"
+                + "UPDATE storageuse SET sizeinbytes=COALESCE(sizeinbytes,0)+" + increment + "\n"
+                + "FROM uptree\n"
+                + "WHERE dvobjectcontainer_id = uptree.id;";
+        
+        int parentsUpdated = em.createNativeQuery(queryString).executeUpdate();
+        // @todo throw an exception if the number of parent dvobjects updated by
+        // the query is < 2 - ? 
+    }
+    
 }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index c5b94b7be58..1b7bff9eeca 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -924,7 +924,8 @@ dataverse.datasize=Total size of the files stored in this dataverse: {0} bytes
 dataverse.storage.quota.allocation=Total quota allocation for this collection: {0} bytes
 dataverse.storage.quota.notdefined=No quota defined for this collection
 dataverse.storage.quota.updated=Storage quota successfully updated for the collection
-dataverse.storage.quota.deleted=Storage quota successfully disabled the collection
+dataverse.storage.quota.deleted=Storage quota successfully disabled for the collection
+dataverse.storage.use=Total recorded size of the files stored in this collection (user-uploaded files plus the versions in the archival tab-delimited format when applicable): {0} bytes
 dataverse.datasize.ioerror=Fatal IO error while trying to determine the total size of the files stored in the dataverse. Please report this error to the Dataverse administrator.
 dataverse.inherited=(inherited from enclosing Dataverse)
 dataverse.default=(Default)
diff --git a/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql b/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
index 5644d06a682..3657642c267 100644
--- a/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
+++ b/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
@@ -1,42 +1,43 @@
--- Adding the storagesize column to the datafile table:
-ALTER TABLE datafile ADD COLUMN IF NOT EXISTS storagesize BIGINT;
 -- The somewhat convoluted queries below populate the storage sizes for the entire
 -- DvObject tree, fast. It IS possible, to do it all with one recursive PostgresQL
 -- query, that will crawl the tree from the leaves (DataFiles) up and add up the
--- sizes for all the Datasets/Collections above. Unfortunately, that takes some hours
--- on a database the size of the one at IQSS. So what we are doing instead is compute
--- the total sizes of all the *directly* linked objects, with 3 linear queries. This
--- will correctly calculate the sizes of all the Datasets (since they can only
--- contain DataFiles, directly, without any extra hierarchy possible) and those
--- Collections that only contain Datasets; but not the sizes of Collections that
--- have sub-collections. To take any sub-collections into account we are then running
--- a recursive query - but then we only need to run it on the tree of Collections only,
--- which should make it manageably fast on any real life instance. 
-----UPDATE dvobject SET storagesize=0;
--- For datafiles, the storage size = main file size by default:
--- (we are excluding any harvested files)
-UPDATE datafile SET storagesize=COALESCE(filesize,0) FROM dataset d, dvobject o WHERE datafile.id = o.id AND o.owner_id = d.id AND d.harvestingclient_id IS null;
+-- sizes for all the Datasets/Collections above. Unfortunately, that appears to take
+-- some hours on a database the size of the one at IQSS. So what we are doing
+-- instead is first compute the total sizes of all the *directly* linked objects,
+-- with a couple of linear queries. This will correctly calculate the sizes of all the
+-- Datasets (since they can only contain DataFiles, without any other hierarchy) and
+-- those Collections that only contain Datasets; but not the sizes of Collections that
+-- have sub-collections. To take any sub-collections into account we will then run
+-- a recursive query - but we only need to run it on the tree of Collections only,
+-- which makes it reasonably fast on any real life instance. 
+-- *Temporarily* add this "tempstoragesize" column to the DvObject table.
+-- It will be used to calculate the storage sizes of all the DvObjectContainers
+-- (Datasets and Collections), as a matter of convenience. Once calculated, the values
+-- will will be moved to the permanent StorageUse table. 
+ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS tempStorageSize BIGINT;
+-- First we calculate the storage size of each individual dataset (a simple sum
+-- of the storage sizes of all the files in the dataset). 
+-- For datafiles, the storage size = main file size by default
+-- (we are excluding any harvested files and datasets):
+UPDATE dvobject SET tempStorageSize=o.combinedStorageSize
+FROM (SELECT datasetobject.id, SUM(file.filesize) AS combinedStorageSize
+FROM dvobject fileobject, dataset datasetobject, datafile file
+WHERE fileobject.owner_id = datasetobject.id
+AND fileobject.id = file.id
+AND datasetobject.harvestingclient_id IS null
+GROUP BY datasetobject.id) o, dataset ds WHERE o.id = dvobject.id AND dvobject.dtype='Dataset' AND dvobject.id = ds.id AND ds.harvestingclient_id IS null;
+
 -- ... but for ingested tabular files the size of the saved original needs to be added, since
 -- those also take space:
 -- (should be safe to assume that there are no *harvested ingested* files)
-UPDATE datafile SET storagesize=datafile.storagesize + COALESCE(datatable.originalFileSize,0) FROM datatable WHERE datatable.datafile_id = datafile.id;
--- *Temporarily* add this "storagesize" column to the DvObject table.
--- It will be used to calculate the storage sizes for all 3 types of DvObjects;
--- a recursive query is used for this purpose further down, and it is convenient
--- to use a column directly in the same table. 
--- Once calculated, the values will will be moved to the permanent locations
--- in the DataFile and StorageUse tables. 
-ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS tempStorageSize BIGINT;
--- Now we can calculate storage sizes of each individual dataset (a simple sum
--- of the storage sizes of all the files in the dataset):
--- (excluding the harvested datasets; this is less important, since there should be
--- significantly fewer datasets than files, but might as well)
-UPDATE dvobject SET tempStorageSize=o.combinedStorageSize
-FROM (SELECT datasetobject.id, SUM(file.storagesize) AS combinedStorageSize
-FROM dvobject fileobject, dvobject datasetobject, datafile file
+UPDATE dvobject SET tempStorageSize=tempStorageSize+o.combinedStorageSize
+FROM (SELECT datasetobject.id, COALESCE(SUM(dt.originalFileSize),0) AS combinedStorageSize
+FROM dvobject fileobject, dvobject datasetobject, datafile file, datatable dt
 WHERE fileobject.owner_id = datasetobject.id
 AND fileobject.id = file.id
+AND dt.datafile_id = file.id
 GROUP BY datasetobject.id) o, dataset ds WHERE o.id = dvobject.id AND dvobject.dtype='Dataset' AND dvobject.id = ds.id AND ds.harvestingclient_id IS null;
+
 -- ... and then we can repeat the same for collections, by setting the storage size
 -- to the sum of the storage sizes of the datasets *directly* in each collection:
 -- (no attemp is made yet to recursively count the sizes all the chilld sub-collections)
@@ -56,7 +57,7 @@ WITH RECURSIVE treestorage (id, owner_id, tempStorageSize, dtype) AS
     FROM dvobject
     WHERE dtype = 'Dataverse'
 
-    UNION
+    UNION ALL
 
     -- Recursive Member:
     SELECT dvobject.id, treestorage.owner_id, dvobject.tempStorageSize, treestorage.dtype
diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java
index 99da9198296..a80adb33b8d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java
+++ b/src/test/java/edu/harvard/iq/dataverse/engine/TestCommandContext.java
@@ -20,6 +20,7 @@
 import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
 import edu.harvard.iq.dataverse.search.savedsearch.SavedSearchServiceBean;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+import edu.harvard.iq.dataverse.storageuse.StorageUseServiceBean;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.workflow.WorkflowServiceBean;
 import java.util.Stack;
@@ -244,6 +245,11 @@ public ConfirmEmailServiceBean confirmEmail() {
     public ActionLogServiceBean actionLog() {
         return null;
     }
+    
+    @Override
+    public StorageUseServiceBean storageUse() {
+        return null;
+    }
 
     @Override
     public void beginCommandSequence() {

From 03e1860f39523a798f65452c339edb9ce20a177f Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 26 Nov 2023 18:14:30 -0500
Subject: [PATCH 789/922] renaming the flyway script, since there is a 6.0.0.3
 in the develop branch already. #8549

---
 ...collection-quotas.sql => V6.0.0.4__8549-collection-quotas.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V6.0.0.3__8549-collection-quotas.sql => V6.0.0.4__8549-collection-quotas.sql} (100%)

diff --git a/src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql b/src/main/resources/db/migration/V6.0.0.4__8549-collection-quotas.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.0.0.3__8549-collection-quotas.sql
rename to src/main/resources/db/migration/V6.0.0.4__8549-collection-quotas.sql

From 8d49fc46d1c4bde6cfe6fc5af30c210201ca9c00 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 26 Nov 2023 19:45:45 -0500
Subject: [PATCH 790/922] a quick stab at a restassured test for the collection
 quotas #8549

---
 src/main/java/propertyFiles/Bundle.properties |  2 +-
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 43 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 12 ++++++
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 1b7bff9eeca..b66d9baa79d 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -923,7 +923,7 @@ dataverse.listing.error=Fatal error trying to list the contents of the dataverse
 dataverse.datasize=Total size of the files stored in this dataverse: {0} bytes
 dataverse.storage.quota.allocation=Total quota allocation for this collection: {0} bytes
 dataverse.storage.quota.notdefined=No quota defined for this collection
-dataverse.storage.quota.updated=Storage quota successfully updated for the collection
+dataverse.storage.quota.updated=Storage quota successfully set for the collection
 dataverse.storage.quota.deleted=Storage quota successfully disabled for the collection
 dataverse.storage.use=Total recorded size of the files stored in this collection (user-uploaded files plus the versions in the archival tab-delimited format when applicable): {0} bytes
 dataverse.datasize.ioerror=Fatal IO error while trying to determine the total size of the files stored in the dataverse. Please report this error to the Dataverse administrator.
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 16726485dee..6bcd935f533 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2362,4 +2362,47 @@ public void testGetHasBeenDeleted() {
         fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data");
         assertTrue(fileHasBeenDeleted);
     }
+    
+    @Test
+    public void testCollectionStorageQuotas() {
+        // A minimal storage quota functionality test: 
+        // - We create a collection and define a storage quota
+        // - We configure Dataverse to enforce it 
+        // - We confirm that we can upload a file with the size under the quota
+        // - We confirm that we cannot upload a file once the quota is reached
+        // - We disable the quota on the collection via the API
+        
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        
+        Response checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
+        checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
+        // This brand new collection shouldn't have any quota defined yet: 
+        assertEquals("No quota defined for this collection", JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        
+        // Set quota to 1K:
+        Response setQuotaResponse = UtilIT.setCollectionQuota(dataverseAlias, 1024, apiToken);
+        setQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
+        assertEquals("Storage quota successfully set for the collection", JsonPath.from(setQuotaResponse.body().asString()).getString("data.message"));
+        
+        // Check again:
+        checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
+        checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
+        assertEquals("Total quota allocation for this collection: 1,024 bytes", JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+
+        UtilIT.enableSetting(SettingsServiceBean.Key.UseStorageQuotas);
+        
+        // Upload a small file: 
+        
+        // [To be continued/work in progress]
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 2336bf8beb8..3cfb625e372 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2983,6 +2983,18 @@ static Response findDataverseStorageSize(String dataverseId, String apiToken) {
                 .get("/api/dataverses/" + dataverseId + "/storagesize");
     }
     
+    static Response checkCollectionQuota(String collectionId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/dataverses/" + collectionId + "/storage/quota");
+    }
+    
+    static Response setCollectionQuota(String collectionId, long allocatedSize, String apiToken) {
+        Response response = given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .post("/api/dataverses/" + collectionId + "/storage/quota" + allocatedSize);
+        return response;
+    }
     
     /**
      * Determine the "payload" storage size of a dataverse

From c9dc1552f39316f7f6178ee069601e66f0f689d1 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 27 Nov 2023 14:11:20 -0500
Subject: [PATCH 791/922] restassured test/work in progress #8549

---
 src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 6bcd935f533..9ad3c353da8 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2403,6 +2403,10 @@ public void testCollectionStorageQuotas() {
         
         // Upload a small file: 
         
+        String pathToFileUnder1K = "src/test/resources/FileRecordJobIT.properties"; 
+        String pathToFileOver1K = "src/test/resources/datacite.xml";
+        
+        
         // [To be continued/work in progress]
     }
 }

From 212baf2ed89f21d9d66da2b5ef6c7652c505141e Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 27 Nov 2023 14:41:43 -0500
Subject: [PATCH 792/922] #9464 return json object as api response

---
 .../DataverseFieldTypeInputLevel.java         |  5 +-
 ...taverseFieldTypeInputLevelServiceBean.java | 10 ++++
 .../iq/dataverse/DataverseServiceBean.java    | 55 ++++++++++++++++---
 .../harvard/iq/dataverse/api/Dataverses.java  |  3 +-
 4 files changed, 61 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java
index c4749be0cb3..a3425987bf8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevel.java
@@ -30,8 +30,9 @@
     @NamedQuery(name = "DataverseFieldTypeInputLevel.findByDataverseIdDatasetFieldTypeId",
             query = "select f from DataverseFieldTypeInputLevel f where f.dataverse.id = :dataverseId and f.datasetFieldType.id = :datasetFieldTypeId"),
     @NamedQuery(name = "DataverseFieldTypeInputLevel.findByDataverseIdAndDatasetFieldTypeIdList",
-            query = "select f from DataverseFieldTypeInputLevel f where f.dataverse.id = :dataverseId and f.datasetFieldType.id in :datasetFieldIdList")
- 
+            query = "select f from DataverseFieldTypeInputLevel f where f.dataverse.id = :dataverseId and f.datasetFieldType.id in :datasetFieldIdList"),
+    @NamedQuery(name = "DataverseFieldTypeInputLevel.findRequiredByDataverseId",
+            query = "select f from DataverseFieldTypeInputLevel f where f.dataverse.id = :dataverseId and f.required = 'true' ")
 })
 @Table(name="DataverseFieldTypeInputLevel"
         ,  uniqueConstraints={
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java
index 66c700f59ce..1bd290ecc4d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseFieldTypeInputLevelServiceBean.java
@@ -88,6 +88,16 @@ public DataverseFieldTypeInputLevel findByDataverseIdDatasetFieldTypeId(Long dat
             return null;
         }         
     }
+    
+    public List<DataverseFieldTypeInputLevel> findRequiredByDataverseId(Long dataverseId) {
+        Query query = em.createNamedQuery("DataverseFieldTypeInputLevel.findRequiredByDataverseId", DataverseFieldTypeInputLevel.class);
+        query.setParameter("dataverseId", dataverseId);
+        try{
+            return query.getResultList();
+        } catch ( NoResultException nre ) {
+            return null;
+        }         
+    }
 
     public void delete(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel) {
         em.remove(em.merge(dataverseFieldTypeInputLevel));
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index bbf35535915..ed46caf65a1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -21,6 +21,7 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 import java.io.File;
 import java.io.IOException;
 import java.sql.Timestamp;
@@ -43,6 +44,8 @@
 import jakarta.persistence.NonUniqueResultException;
 import jakarta.persistence.PersistenceContext;
 import jakarta.persistence.TypedQuery;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.everit.json.schema.Schema;
@@ -931,12 +934,9 @@ public List<Object[]> getDatasetTitlesWithinDataverse(Long dataverseId) {
 
         
     public  String getCollectionDatasetSchema(String dataverseAlias) {
-       
-        List<MetadataBlock> selectedBlocks = new ArrayList<>();
-        List<DatasetFieldType> requiredDSFT = new ArrayList<>();
-
+        
         Dataverse testDV = this.findByAlias(dataverseAlias);
-
+        
         while (!testDV.isMetadataBlockRoot()) {
             if (testDV.getOwner() == null) {
                 break; // we are at the root; which by defintion is metadata blcok root, regarldess of the value
@@ -944,6 +944,25 @@ public  String getCollectionDatasetSchema(String dataverseAlias) {
             testDV = testDV.getOwner();
         }
         
+        /* Couldn't get the 'return base if no extra required fields to work with the path provided
+        leaving it as 'out of scope' for now SEK 11/27/2023
+
+        List<DataverseFieldTypeInputLevel> required = new ArrayList<>();
+
+        required = dataverseFieldTypeInputLevelService.findRequiredByDataverseId(testDV.getId());
+        
+        if (required == null || required.isEmpty()){
+            String pathToJsonFile = "src/main/resources/edu/harvas/iq/dataverse/baseDatasetSchema.json";
+            String baseSchema = getBaseSchemaStringFromFile(pathToJsonFile);
+            if (baseSchema != null && !baseSchema.isEmpty()){
+                return baseSchema;
+            }
+        }
+        
+        */
+        List<MetadataBlock> selectedBlocks = new ArrayList<>();
+        List<DatasetFieldType> requiredDSFT = new ArrayList<>();
+        
         selectedBlocks.addAll(testDV.getMetadataBlocks());
 
         for (MetadataBlock mdb : selectedBlocks) {
@@ -992,9 +1011,13 @@ public  String getCollectionDatasetSchema(String dataverseAlias) {
                 }
             }
         }
-        
+        int countMDB = 0;
         for (MetadataBlock mdb : hasReqFields) {
+            if (countMDB>0){
+                retval += ",";
+            }
             retval += getCustomMDBSchema(mdb, requiredDSFT);
+            countMDB++;            
         }
         
         retval += "\n                     }";
@@ -1037,7 +1060,7 @@ private String getCustomMDBSchema (MetadataBlock mdb, List<DatasetFieldType> req
         }
         
         }
-
+        
         return retval;
     }
     
@@ -1058,6 +1081,20 @@ public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
         return BundleUtil.getStringFromBundle("dataverses.api.validate.json.succeeded");
     }
     
+    static String getBaseSchemaStringFromFile(String pathToJsonFile) {
+        File datasetSchemaJson = new File(pathToJsonFile);
+        try {
+            String datasetSchemaAsJson = new String(Files.readAllBytes(Paths.get(datasetSchemaJson.getAbsolutePath())));
+            return datasetSchemaAsJson;
+        } catch (IOException ex) {
+            logger.info("IO - failed to get schema file  - will build on fly " +ex.getMessage());
+            return null;
+        } catch (Exception e){
+            logger.info("Other exception - failed to get schema file  - will build on fly. " + e.getMessage());
+            return null;
+        }
+    }
+    
     private  String datasetSchemaPreface = 
     "{\n" +
     "    \"$schema\": \"http://json-schema.org/draft-04/schema#\",\n" +
@@ -1140,7 +1177,7 @@ public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
 "                                }\n" +
 "                            },\n" +
 "                            \"required\": [\"fields\"]\n" +
-"                        },";
+"                        }";
     
     private String endOfjson = ",\n" +
 "                    \"required\": [blockNames]\n" +
@@ -1152,5 +1189,5 @@ public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
 "    \"required\": [\"datasetVersion\"]\n" +
 "}\n";
     
-            
+    
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 557b7df202b..c6ae619f36a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -258,7 +258,8 @@ public Response getDatasetSchema(@Context ContainerRequestContext crc, @PathPara
 
         try {
             String datasetSchema = execCommand(new GetDatasetSchemaCommand(createDataverseRequest(u), findDataverseOrDie(idtf)));
-            return ok(datasetSchema);
+            JsonObject jsonObject = JsonUtil.getJsonObject(datasetSchema);
+            return ok(jsonObject);
         } catch (WrappedResponse ex) {
             Logger.getLogger(Dataverses.class.getName()).log(Level.SEVERE, null, ex);
             return ex.getResponse();

From 93670260de18059ff75a2fa3f8ca00e395b64110 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 27 Nov 2023 14:46:30 -0500
Subject: [PATCH 793/922] #9464 revert harvesting changes made in error

---
 .../dataverse/metrics/MetricsServiceBean.java | 51 +++++++++----------
 .../harvard/iq/dataverse/api/MetricsIT.java   | 14 ++---
 2 files changed, 29 insertions(+), 36 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
index a50797fe443..1b5619c53e0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -138,8 +138,8 @@ public JsonArray getDatasetsTimeSeries(UriInfo uriInfo, String dataLocation, Dat
                 + "from datasetversion\n"
                 + "where versionstate='RELEASED' \n"
                 + (((d == null)&&(DATA_LOCATION_ALL.equals(dataLocation))) ? "" : "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id=dvobject.id\n")
-                + ((DATA_LOCATION_LOCAL.equals(dataLocation)) ? "and dvobject.harvestingclient_id IS NULL and publicationdate is not null\n " : "")
-                + ((DATA_LOCATION_REMOTE.equals(dataLocation)) ? "and dvobject.harvestingclient_id IS NOT NULL\n "  : "")
+                + ((DATA_LOCATION_LOCAL.equals(dataLocation)) ? "and dataset.harvestingclient_id IS NULL and publicationdate is not null\n " : "")
+                + ((DATA_LOCATION_REMOTE.equals(dataLocation)) ? "and dataset.harvestingclient_id IS NOT NULL\n "  : "")
                 + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n ")
                 + (((d == null)&&(DATA_LOCATION_ALL.equals(dataLocation))) ? "" : ")\n")
                 + "group by dataset_id) as subq group by subq.date order by date;"
@@ -156,11 +156,11 @@ public JsonArray getDatasetsTimeSeries(UriInfo uriInfo, String dataLocation, Dat
      * @param d
      */
     public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
-        String dataLocationLine = "(date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM') and dvobject.harvestingclient_id IS NULL)\n";
+        String dataLocationLine = "(date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM') and dataset.harvestingclient_id IS NULL)\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
             //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
-            String harvestBaseLine = "(date_trunc('month', createtime) <=  to_date('" + yyyymm + "','YYYY-MM') and dvobject.harvestingclient_id IS NOT NULL)\n";
+            String harvestBaseLine = "(date_trunc('month', createtime) <=  to_date('" + yyyymm + "','YYYY-MM') and dataset.harvestingclient_id IS NOT NULL)\n";
             if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
                 dataLocationLine = harvestBaseLine; // replace
             } else if (DATA_LOCATION_ALL.equals(dataLocation)) {
@@ -189,7 +189,7 @@ public long datasetsToMonth(String yyyymm, String dataLocation, Dataverse d) {
                         + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
                         + "from datasetversion\n"
                         + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                        + "join dvobject on dvobject.id = dataset.id\n"
+                        + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
                         + "where versionstate='RELEASED' \n"
                         + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n ")
                         + "and \n"
@@ -207,17 +207,16 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
         // A published local datasets may have more than one released version!
         // So that's why we have to jump through some extra hoops below
         // in order to select the latest one:
-        String originClause = "(datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in\n"
-                + "(\n"
-                + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n"
-                + "       from datasetversion\n"
-                + "       join dataset on dataset.id = datasetversion.dataset_id\n"
-                + "       join dvobject on dataset.id = dvobject.id\n"
-                + "       where versionstate='RELEASED'\n"
-                + "       and dvobject.harvestingclient_id is null"
-                + "       and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
-                + "       group by dataset_id\n"
-                + "))\n";
+        String originClause = "(datasetversion.dataset_id || ':' || datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber) in\n" +
+                "(\n" +
+                "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))\n" +
+                "       from datasetversion\n" +
+                "       join dataset on dataset.id = datasetversion.dataset_id\n" +
+                "       where versionstate='RELEASED'\n" +
+                "       	     and dataset.harvestingclient_id is null\n" +
+                "       	     and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n" +
+                "       group by dataset_id\n" +
+                "))\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
             //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
@@ -226,7 +225,7 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
             // so the query is simpler:
             String harvestOriginClause = "(\n" +
                     "   datasetversion.dataset_id = dataset.id\n" +
-                    "   AND dvobject.harvestingclient_id IS NOT null \n" +
+                    "   AND dataset.harvestingclient_id IS NOT null \n" +
                     "   AND date_trunc('month', datasetversion.createtime) <=  to_date('" + yyyymm + "','YYYY-MM')\n" +
                     ")\n";
 
@@ -245,7 +244,7 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
                 + "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n"
                 + "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n"
                 + "JOIN dataset ON dataset.id = datasetversion.dataset_id\n"
-                + "JOIN dvobject ON dvobject.id = dataset.id\n"
+                + ((d == null) ? "" : "JOIN dvobject ON dvobject.id = dataset.id\n")
                 + "WHERE\n"
                 + originClause
                 + "AND datasetfieldtype.name = 'subject'\n"
@@ -259,11 +258,11 @@ public List<Object[]> datasetsBySubjectToMonth(String yyyymm, String dataLocatio
     }
 
     public long datasetsPastDays(int days, String dataLocation, Dataverse d) {
-        String dataLocationLine = "(releasetime > current_date - interval '" + days + "' day and dvobject.harvestingclient_id IS NULL)\n";
+        String dataLocationLine = "(releasetime > current_date - interval '" + days + "' day and dataset.harvestingclient_id IS NULL)\n";
 
         if (!DATA_LOCATION_LOCAL.equals(dataLocation)) { // Default api state is DATA_LOCATION_LOCAL
             //we have to use createtime for harvest as post dvn3 harvests do not have releasetime populated
-            String harvestBaseLine = "(createtime > current_date - interval '" + days + "' day and dvobject.harvestingclient_id IS NOT NULL)\n";
+            String harvestBaseLine = "(createtime > current_date - interval '" + days + "' day and dataset.harvestingclient_id IS NOT NULL)\n";
             if (DATA_LOCATION_REMOTE.equals(dataLocation)) {
                 dataLocationLine = harvestBaseLine; // replace
             } else if (DATA_LOCATION_ALL.equals(dataLocation)) {
@@ -277,7 +276,7 @@ public long datasetsPastDays(int days, String dataLocation, Dataverse d) {
                         + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max\n"
                         + "from datasetversion\n"
                         + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                        + "join dvobject on dvobject.id = dataset.id\n"
+                        + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
                         + "where versionstate='RELEASED' \n"
                         + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                         + "and \n"
@@ -305,7 +304,7 @@ public JsonArray filesTimeSeries(Dataverse d) {
                         + "where datasetversion.id=filemetadata.datasetversion_id\n"
                         + "and versionstate='RELEASED' \n"
                         + "and dataset_id in (select dataset.id from dataset, dvobject where dataset.id=dvobject.id\n"
-                        + "and dvobject.harvestingclient_id IS NULL and publicationdate is not null\n "
+                        + "and dataset.harvestingclient_id IS NULL and publicationdate is not null\n "
                         + ((d == null) ? ")" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + "))\n ")
                         + "group by filemetadata.id) as subq group by subq.date order by date;");
         logger.log(Level.FINE, "Metric query: {0}", query);
@@ -328,11 +327,11 @@ public long filesToMonth(String yyyymm, Dataverse d) {
                 + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                +  "join dvobject on dvobject.id = dataset.id\n"
+                + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
                 + "where versionstate='RELEASED'\n"
                 + ((d == null) ? "" : "and dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
                 + "and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
-                + "and dvobject.harvestingclient_id is null\n"
+                + "and dataset.harvestingclient_id is null\n"
                 + "group by dataset_id \n"
                 + ");"
         );
@@ -351,11 +350,11 @@ public long filesPastDays(int days, Dataverse d) {
                 + "select datasetversion.dataset_id || ':' || max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) as max \n"
                 + "from datasetversion\n"
                 + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                + "join dvobject on dvobject.id = dataset.id\n"
+                + ((d == null) ? "" : "join dvobject on dvobject.id = dataset.id\n")
                 + "where versionstate='RELEASED'\n"
                 + "and releasetime > current_date - interval '" + days + "' day\n"
                 + ((d == null) ? "" : "AND dvobject.owner_id in (" + getCommaSeparatedIdStringForSubtree(d, "Dataverse") + ")\n")
-                + "and dvobject.harvestingclient_id is null\n"
+                + "and dataset.harvestingclient_id is null\n"
                 + "group by dataset_id \n"
                 + ");"
         );
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java
index b961a86dc0b..e3328eefb4a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/MetricsIT.java
@@ -30,7 +30,7 @@ public static void cleanUpClass() {
 
     @Test
     public void testGetDataversesToMonth() {
-        String yyyymm = "2023-04";
+        String yyyymm = "2018-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsDataversesToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -54,7 +54,7 @@ public void testGetDataversesToMonth() {
 
     @Test
     public void testGetDatasetsToMonth() {
-        String yyyymm = "2023-04";
+        String yyyymm = "2018-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsDatasetsToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -77,7 +77,7 @@ public void testGetDatasetsToMonth() {
 
     @Test
     public void testGetFilesToMonth() {
-        String yyyymm = "2023-04";
+        String yyyymm = "2018-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsFilesToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -100,7 +100,7 @@ public void testGetFilesToMonth() {
 
     @Test
     public void testGetDownloadsToMonth() {
-        String yyyymm = "2023-04";
+        String yyyymm = "2018-04";
 //        yyyymm = null;
         Response response = UtilIT.metricsDownloadsToMonth(yyyymm, null);
         String precache = response.prettyPrint();
@@ -283,12 +283,6 @@ public void testGetDatasetsBySubject() {
         response = UtilIT.metricsDatasetsBySubject("dataLocation=local");
         response.then().assertThat()
                 .statusCode(OK.getStatusCode());
-        
-        //Test ok when passing remote
-        response = UtilIT.metricsDatasetsBySubject("dataLocation=remote");
-        response.prettyPrint();
-        response.then().assertThat()
-                .statusCode(OK.getStatusCode());
     }
 
     @Test

From 808efdd7cf08c3ec4955dd25d8bbd1078c3f09db Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Mon, 27 Nov 2023 14:54:25 -0500
Subject: [PATCH 794/922] fixt east/west mixup

---
 .../harvard/iq/dataverse/DatasetFieldValueValidator.java  | 2 +-
 src/main/java/propertyFiles/Bundle.properties             | 2 +-
 .../iq/dataverse/DatasetFieldValueValidatorTest.java      | 8 ++++----
 .../harvard/iq/dataverse/search/IndexServiceBeanTest.java | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
index 7941fa5efa1..23301bf86a1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
@@ -274,7 +274,7 @@ public static boolean validateBoundingBox(final String westLon, final String eas
             Float east = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.eastLongitude, eastLon);
             Float north = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.northLatitude, northLat);
             Float south = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.southLatitude, southLat);
-            returnVal = east < west && south < north;
+            returnVal = west < east && south < north;
         } catch (IllegalArgumentException e) {
             returnVal = false;
         }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 15b0ebb8020..4db66006a80 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1623,7 +1623,7 @@ dataset.metadata.alternativePersistentId.tip=A previously used persistent identi
 dataset.metadata.invalidEntry=is not a valid entry.
 dataset.metadata.invalidDate=is not a valid date. "yyyy" is a supported format.
 dataset.metadata.invalidNumber=is not a valid number.
-dataset.metadata.invalidGeospatialCoordinates=has invalid coordinates. West must be greater than East and North must be greater than South.
+dataset.metadata.invalidGeospatialCoordinates=has invalid coordinates. East must be greater than West and North must be greater than South.
 dataset.metadata.invalidInteger=is not a valid integer.
 dataset.metadata.invalidURL=is not a valid URL.
 dataset.metadata.invalidEmail=is not a valid email address.
diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
index 2729e5b042a..37274f5306a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
@@ -216,12 +216,12 @@ public void testInvalidEmail() {
     @Test
     public void testBoundingBoxValidity() {
         // valid tests
-        assertTrue(DatasetFieldValueValidator.validateBoundingBox("180", "-180", "90", "-90"));
-        assertTrue(DatasetFieldValueValidator.validateBoundingBox("180", "-180", "90", null));
+        assertTrue(DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", "-90"));
+        assertTrue(DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", null));
 
         // invalid tests
-        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("180", "-180", "90", "junk"));
-        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("40", "45", "90", "0"));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", "junk"));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("45", "40", "90", "0"));
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("360", "0", "90", "-90"));
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
index a0330e1fcad..adf48e05f09 100644
--- a/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/search/IndexServiceBeanTest.java
@@ -110,8 +110,8 @@ public void testValidateBoundingBox() throws SolrServerException, IOException {
         DatasetFieldCompoundValue val = new DatasetFieldCompoundValue();
         val.setParentDatasetField(dsf);
         val.setChildDatasetFields(Arrays.asList(
-                constructBoundingBoxValue(DatasetFieldConstant.westLongitude, "34.8"),
-                constructBoundingBoxValue(DatasetFieldConstant.eastLongitude, "34.9"), // bad value. must be less than west
+                constructBoundingBoxValue(DatasetFieldConstant.westLongitude, "34.9"), // bad value. must be less than east
+                constructBoundingBoxValue(DatasetFieldConstant.eastLongitude, "34.8"),
                 constructBoundingBoxValue(DatasetFieldConstant.northLatitude, "34.2"),
                 constructBoundingBoxValue(DatasetFieldConstant.southLatitude, "34.1")
         ));

From b7a3e7803dd6689c0060fdd2d5a2dab5e00a51ff Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 27 Nov 2023 14:29:01 -0500
Subject: [PATCH 795/922] add dataset JSON Schema to API guide, add test #9464

---
 .../source/_static/api/dataset-schema.json    | 123 ++++++++++++++++++
 doc/sphinx-guides/source/api/native-api.rst   |   2 +
 .../harvard/iq/dataverse/api/DatasetsIT.java  |   9 ++
 3 files changed, 134 insertions(+)
 create mode 100644 doc/sphinx-guides/source/_static/api/dataset-schema.json

diff --git a/doc/sphinx-guides/source/_static/api/dataset-schema.json b/doc/sphinx-guides/source/_static/api/dataset-schema.json
new file mode 100644
index 00000000000..a66ab43bd0f
--- /dev/null
+++ b/doc/sphinx-guides/source/_static/api/dataset-schema.json
@@ -0,0 +1,123 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$defs": {
+    "field": {
+        "type": "object",
+        "required": ["typeClass", "multiple", "typeName"],
+        "properties": {
+            "value": {
+                "anyOf": [
+                    {
+                        "type": "array"
+                    },
+                    {
+                        "type": "string"
+                    },
+                    {
+                        "$ref": "#/$defs/field"
+                    }
+                ]
+            },
+            "typeClass": {
+                "type": "string"
+            },
+            "multiple": {
+                "type": "boolean"
+            },
+            "typeName": {
+                "type": "string"
+            }
+        }
+    }
+},
+"type": "object",
+"properties": {
+    "datasetVersion": {
+        "type": "object",
+        "properties": {
+           "license": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "uri": {
+                        "type": "string",
+                        "format": "uri"
+                   }
+                },
+                "required": ["name", "uri"]
+            },
+            "metadataBlocks": {
+                "type": "object",
+               "properties": {
+                           "citation": {
+                            "type": "object",
+                            "properties": {
+                                "fields": {
+                                    "type": "array",
+                                    "items": {
+                                        "$ref": "#/$defs/field"
+                                    },
+                                    "minItems": 5,
+                                    "allOf": [
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "title"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "author"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "datasetContact"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "dsDescription"
+                                                    }
+                                                }
+                                            }
+                                        },
+                                        {
+                                            "contains": {
+                                                "properties": {
+                                                    "typeName": {
+                                                        "const": "subject"
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    ]
+                                }
+                            },
+                            "required": ["fields"]
+                        }
+                     },
+                    "required": ["citation"]
+                }
+            },
+            "required": ["metadataBlocks"]
+        }
+    },
+    "required": ["datasetVersion"]
+}
+
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 61b1a9443b2..ca043af39e3 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -528,6 +528,8 @@ The fully expanded example above (without environment variables) looks like this
 
 Note: you must have Add Dataset permission in the given Dataverse collection to invoke this endpoint.
 
+While it is recommended to download a copy of the JSON Schema from the collection (as above) to account for any fields that have been marked as required, you can also download a minimal :download:`dataset-schema.json <../_static/api/dataset-schema.json>` to get a sense of the schema when no customizations have been made.
+
 .. _validate-dataset-json:
 
 Validate Dataset.json file for a Collection
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 9fe08498de9..6a746b7c5b5 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -62,6 +62,7 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import edu.harvard.iq.dataverse.util.json.JSONLDUtil;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
 import java.io.File;
 import java.io.IOException;
@@ -179,6 +180,14 @@ public void testCollectionSchema(){
         getCollectionSchemaResponse.prettyPrint();
         getCollectionSchemaResponse.then().assertThat()
                 .statusCode(200);
+
+        JsonObject expectedSchema = null;
+        try {
+            expectedSchema = JsonUtil.getJsonObjectFromFile("doc/sphinx-guides/source/_static/api/dataset-schema.json");
+        } catch (IOException ex) {
+        }
+
+        assertEquals(JsonUtil.prettyPrint(expectedSchema), JsonUtil.prettyPrint(getCollectionSchemaResponse.body().asString()));
         
         String expectedJson = UtilIT.getDatasetJson("scripts/search/tests/data/dataset-finch1.json");
         

From 2d3f7aba2bc180c8d151e076a4f476d039cc27f3 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 27 Nov 2023 15:45:12 -0500
Subject: [PATCH 796/922] just return the JSON Schema, don't wrap in "data,
 message" #9464

---
 src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index c6ae619f36a..202a54a9e77 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -259,7 +259,7 @@ public Response getDatasetSchema(@Context ContainerRequestContext crc, @PathPara
         try {
             String datasetSchema = execCommand(new GetDatasetSchemaCommand(createDataverseRequest(u), findDataverseOrDie(idtf)));
             JsonObject jsonObject = JsonUtil.getJsonObject(datasetSchema);
-            return ok(jsonObject);
+            return Response.ok(jsonObject).build();
         } catch (WrappedResponse ex) {
             Logger.getLogger(Dataverses.class.getName()).log(Level.SEVERE, null, ex);
             return ex.getResponse();

From 0a77e2a938510d79645ad3ce5fcd349da3e8495f Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 27 Nov 2023 15:53:06 -0500
Subject: [PATCH 797/922] tweak docs #9464

---
 doc/sphinx-guides/source/api/native-api.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index ca043af39e3..bd49f945d4e 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -507,10 +507,10 @@ The fully expanded example above (without environment variables) looks like this
 
 .. _get-dataset-json-schema:
 
-Retrieve a JSON schema for a Collection
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Retrieve a Dataset JSON Schema for a Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Retrieves a JSON schema customized for a given Dataverse collection in order to validate a Dataset JSON file prior to creating the dataset:
+Retrieves a JSON schema customized for a given collection in order to validate a dataset JSON file prior to creating the dataset:
 
 .. code-block:: bash
 
@@ -526,16 +526,16 @@ The fully expanded example above (without environment variables) looks like this
 
   curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/datasetSchema"
 
-Note: you must have Add Dataset permission in the given Dataverse collection to invoke this endpoint.
+Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint.
 
 While it is recommended to download a copy of the JSON Schema from the collection (as above) to account for any fields that have been marked as required, you can also download a minimal :download:`dataset-schema.json <../_static/api/dataset-schema.json>` to get a sense of the schema when no customizations have been made.
 
 .. _validate-dataset-json:
 
-Validate Dataset.json file for a Collection
+Validate Dataset JSON File for a Collection
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Validates a Dataset json file customized for a given Dataverse collection prior to creating the dataset:
+Validates a dataset JSON file customized for a given collection prior to creating the dataset:
 
 .. code-block:: bash
 
@@ -543,15 +543,15 @@ Validates a Dataset json file customized for a given Dataverse collection prior
   export SERVER_URL=https://demo.dataverse.org
   export ID=root
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/validateDatasetJson" --upload-file dataset.json -H 'Content-type:application/json'
+  curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/validateDatasetJson" -H 'Content-type:application/json' --upload-file dataset.json
 
 The fully expanded example above (without environment variables) looks like this:
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/validateDatasetJson" --upload-file dataset.json -H 'Content-type:application/json'
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/validateDatasetJson" -H 'Content-type:application/json' --upload-file dataset.json
 
-Note: you must have Add Dataset permission in the given Dataverse collection to invoke this endpoint.
+Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint.
 
 .. _create-dataset-command: 
 

From 7db36293364118244e0582c3d529a34cd5b66395 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 27 Nov 2023 15:57:24 -0500
Subject: [PATCH 798/922] removing trailing newline #9464

---
 doc/sphinx-guides/source/_static/api/dataset-schema.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/doc/sphinx-guides/source/_static/api/dataset-schema.json b/doc/sphinx-guides/source/_static/api/dataset-schema.json
index a66ab43bd0f..34b8a1eeedb 100644
--- a/doc/sphinx-guides/source/_static/api/dataset-schema.json
+++ b/doc/sphinx-guides/source/_static/api/dataset-schema.json
@@ -120,4 +120,3 @@
     },
     "required": ["datasetVersion"]
 }
-

From 194945b36bd052569e6699c53a78de839d2c1f23 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 28 Nov 2023 09:16:50 -0500
Subject: [PATCH 799/922] remove cruft (unused) #9464

---
 .../iq/dataverse/baseDatasetSchema.json       | 123 ------------------
 1 file changed, 123 deletions(-)
 delete mode 100644 src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json

diff --git a/src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json b/src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json
deleted file mode 100644
index a37e216b2ea..00000000000
--- a/src/main/resources/edu/harvard/iq/dataverse/baseDatasetSchema.json
+++ /dev/null
@@ -1,123 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-04/schema#",
-    "$defs": {
-    "field": {
-        "type": "object",
-        "required": ["typeClass", "multiple", "typeName"],
-        "properties": {
-            "value": {
-                "anyOf": [
-                    {
-                        "type": "array"
-                    },
-                    {
-                        "type": "string"
-                    },
-                    {
-                        "$ref": "#/$defs/field"
-                    }
-                ]
-            },
-            "typeClass": {
-                "type": "string"
-            },
-            "multiple": {
-                "type": "boolean"
-            },
-            "typeName": {
-                "type": "string"
-            }
-        }
-    }
-},
-"type": "object",
-"properties": {
-    "datasetVersion": {
-        "type": "object",
-        "properties": {
-           "license": {
-                "type": "object",
-                "properties": {
-                    "name": {
-                        "type": "string"
-                    },
-                    "uri": {
-                        "type": "string",
-                        "format": "uri"
-                   }
-                },
-                "required": ["name", "uri"]
-            },
-            "metadataBlocks": {
-                "type": "object",
-                "properties": {
-                           "citation": {
-                            "type": "object",
-                            "properties": {
-                                "fields": {
-                                    "type": "array",
-                                    "items": {
-                                        "$ref": "#/$defs/field"
-                                    },
-                                    "minItems": 5,
-                                    "allOf": [
-                                        {
-                                            "contains": {
-                                                "properties": {
-                                                    "typeName": {
-                                                        "const": "title"
-                                                    }
-                                                }
-                                            }
-                                        },
-                                        {
-                                            "contains": {
-                                                "properties": {
-                                                    "typeName": {
-                                                        "const": "author"
-                                                    }
-                                                }
-                                            }
-                                        },
-                                        {
-                                            "contains": {
-                                                "properties": {
-                                                    "typeName": {
-                                                        "const": "datasetContact"
-                                                    }
-                                                }
-                                            }
-                                        },
-                                        {
-                                            "contains": {
-                                                "properties": {
-                                                    "typeName": {
-                                                        "const": "dsDescription"
-                                                    }
-                                                }
-                                            }
-                                        },
-                                        {
-                                            "contains": {
-                                                "properties": {
-                                                    "typeName": {
-                                                        "const": "subject"
-                                                    }
-                                                }
-                                            }
-                                        }
-                                    ]
-                                }
-                            },
-                            "required": ["fields"]
-                        }
-                    },
-                    "required": ["citation"]
-                }
-            },
-            "required": ["metadataBlocks"]
-        }
-    },
-    "required": ["datasetVersion"]
-}
-

From c1bd009b8d2ebc297a51e6d23358ab76137c3848 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 28 Nov 2023 09:18:57 -0500
Subject: [PATCH 800/922] format code (no-op) #9464

---
 .../java/edu/harvard/iq/dataverse/api/UtilIT.java    | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index b3f8d639721..cd05719402f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -406,26 +406,22 @@ static Response getGuestbookResponses(String dataverseAlias, Long guestbookId, S
         }
         return requestSpec.get("/api/dataverses/" + dataverseAlias + "/guestbookResponses/");
     }
-    
-    static Response getCollectionSchema (String dataverseAlias, String apiToken){
-        
+
+    static Response getCollectionSchema(String dataverseAlias, String apiToken) {
         Response getCollectionSchemaResponse = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json")
                 .get("/api/dataverses/" + dataverseAlias + "/datasetSchema");
         return getCollectionSchemaResponse;
-        
     }
-    
-    static Response validateDatasetJson (String dataverseAlias, String datasetJson, String apiToken){
-        
+
+    static Response validateDatasetJson(String dataverseAlias, String datasetJson, String apiToken) {
         Response getValidateDatasetJsonResponse = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .body(datasetJson)
                 .contentType("application/json")
                 .post("/api/dataverses/" + dataverseAlias + "/validateDatasetJson");
         return getValidateDatasetJsonResponse;
-        
     }
 
     static Response createRandomDatasetViaNativeApi(String dataverseAlias, String apiToken) {

From c4d9b6e4a9741d07cc9193e794455c35a08320fd Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 28 Nov 2023 09:26:34 -0500
Subject: [PATCH 801/922] add new endpoints to API changelog #9464

---
 doc/sphinx-guides/source/api/changelog.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index 1726736e75c..dd2d129a420 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -8,6 +8,11 @@ API Changelog
 6.1
 ---
 
+New
+~~~
+- **/api/dataverses/{id}/datasetSchema**: See :ref:`get-dataset-json-schema`.
+- **/api/dataverses/{id}/validateDatasetJson**: See :ref:`validate-dataset-json`.
+
 Changes
 ~~~~~~~
 - **/api/datasets/{id}/versions/{versionId}/citation**: This endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation. See :ref:`get-citation`.

From 45df764c3b9c8d8f504f1134fb302bb4ea10a6bd Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 28 Nov 2023 09:28:30 -0500
Subject: [PATCH 802/922] tweak release note #9464

---
 doc/release-notes/9464-json-validation.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/release-notes/9464-json-validation.md b/doc/release-notes/9464-json-validation.md
index 3703b420225..4b08f2ca9dd 100644
--- a/doc/release-notes/9464-json-validation.md
+++ b/doc/release-notes/9464-json-validation.md
@@ -1,3 +1,3 @@
-Functionality has been added to help validate dataset json prior to dataset creation. There are two new API endpoints in this release. The first takes in a Dataverse Collection alias and returns a custom schema based on the required fields of the collection.
-The second takes in a Dataverse collection alias and a dataset json file and does an automated validation of the json file against the custom schema for the collection. (Issue 9464 and 9465)
+Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. (Issue #9464 and #9465)
 
+For documentation see the API changelog: http://preview.guides.gdcc.io/en/develop/api/changelog.html

From d8e327d8c56793f24213385b9dcca864b2a134e3 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 28 Nov 2023 09:46:16 -0500
Subject: [PATCH 803/922] add "v" to make anchor links meaningful #9464 #10060

Anchors like #v6.1 are much better than #id1, which is meaningless.
---
 doc/sphinx-guides/source/api/changelog.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index dd2d129a420..cbf1cb329b9 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -5,8 +5,8 @@ API Changelog
     :local:
     :depth: 1
 
-6.1
----
+v6.1
+----
 
 New
 ~~~
@@ -17,8 +17,8 @@ Changes
 ~~~~~~~
 - **/api/datasets/{id}/versions/{versionId}/citation**: This endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation. See :ref:`get-citation`.
 
-6.0
----
+v6.0
+----
 
 Changes
 ~~~~~~~

From 8e69c6d8d7b995607d121a87561f9850ce387ae0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 28 Nov 2023 10:29:12 -0500
Subject: [PATCH 804/922] remove new `response()` method, use regular `ok()`
 instead #9466

---
 .../edu/harvard/iq/dataverse/api/AbstractApiBean.java  | 10 ----------
 src/main/java/edu/harvard/iq/dataverse/api/Info.java   | 10 +++++-----
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
index 754ea95e427..027f9e0fcb1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java
@@ -636,16 +636,6 @@ protected Response response(DataverseRequestHandler hdl, User user) {
         }
     }
 
-    /***
-     * The preferred way of handling a request from an open API.
-     *
-     * @param hdl handling code block.
-     * @return HTTP Response appropriate for the way {@code hdl} executed.
-     */
-    protected Response response(DataverseRequestHandler hdl) {
-        return response(hdl, null);
-    }
-
     private Response handleDataverseRequestHandlerException(Exception ex) {
         String incidentId = UUID.randomUUID().toString();
         logger.log(Level.SEVERE, "API internal error " + incidentId +": " + ex.getMessage(), ex);
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index 5478c3be0bd..40ce6cd25b7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -38,21 +38,21 @@ public Response getInfo() {
         String[] comps = versionStr.split("build",2);
         String version = comps[0].trim();
         JsonValue build = comps.length > 1 ? Json.createArrayBuilder().add(comps[1].trim()).build().get(0) : JsonValue.NULL;
-
-        return response( req -> ok( Json.createObjectBuilder().add("version", version)
-                                                              .add("build", build)));
+        return ok(Json.createObjectBuilder()
+                .add("version", version)
+                .add("build", build));
     }
 
     @GET
     @Path("server")
     public Response getServer() {
-        return response( req -> ok(JvmSettings.FQDN.lookup()));
+        return ok(JvmSettings.FQDN.lookup());
     }
 
     @GET
     @Path("apiTermsOfUse")
     public Response getTermsOfUse() {
-        return response( req -> ok(systemConfig.getApiTermsOfUse()));
+        return ok(systemConfig.getApiTermsOfUse());
     }
 
     @GET

From 78404cef91d20f0f41753866e4ee156d51cd3a69 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Tue, 28 Nov 2023 11:18:26 -0500
Subject: [PATCH 805/922] fixt box validation for misisng values

---
 .../harvard/iq/dataverse/DatasetFieldValueValidator.java  | 8 ++++----
 src/main/java/propertyFiles/Bundle.properties             | 2 +-
 .../iq/dataverse/DatasetFieldValueValidatorTest.java      | 4 +++-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
index 23301bf86a1..821497299a9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
@@ -274,7 +274,7 @@ public static boolean validateBoundingBox(final String westLon, final String eas
             Float east = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.eastLongitude, eastLon);
             Float north = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.northLatitude, northLat);
             Float south = verifyBoundingBoxCoordinatesWithinRange(DatasetFieldConstant.southLatitude, southLat);
-            returnVal = west < east && south < north;
+            returnVal = west <= east && south <= north;
         } catch (IllegalArgumentException e) {
             returnVal = false;
         }
@@ -286,9 +286,9 @@ private static Float verifyBoundingBoxCoordinatesWithinRange(final String name,
         int max = name.equals(DatasetFieldConstant.westLongitude) || name.equals(DatasetFieldConstant.eastLongitude) ? 180 : 90;
         int min = max * -1;
 
-        final Float returnVal = value != null ? Float.parseFloat(value) : min; // defaults to min if value is missing
-        if (returnVal < min || returnVal > max) {
-            throw new IllegalArgumentException(String.format("Value (%s) not in range (%s-%s)", returnVal, min, max));
+        final Float returnVal = value != null ? Float.parseFloat(value) : Float.NaN;
+        if (returnVal.isNaN() || returnVal < min || returnVal > max) {
+            throw new IllegalArgumentException(String.format("Value (%s) not in range (%s-%s)", returnVal.isNaN() ? "missing" : returnVal, min, max));
         }
         return returnVal;
     }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 4db66006a80..6b2c30f2a4d 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -1623,7 +1623,7 @@ dataset.metadata.alternativePersistentId.tip=A previously used persistent identi
 dataset.metadata.invalidEntry=is not a valid entry.
 dataset.metadata.invalidDate=is not a valid date. "yyyy" is a supported format.
 dataset.metadata.invalidNumber=is not a valid number.
-dataset.metadata.invalidGeospatialCoordinates=has invalid coordinates. East must be greater than West and North must be greater than South.
+dataset.metadata.invalidGeospatialCoordinates=has invalid coordinates. East must be greater than West and North must be greater than South. Missing values are NOT allowed.
 dataset.metadata.invalidInteger=is not a valid integer.
 dataset.metadata.invalidURL=is not a valid URL.
 dataset.metadata.invalidEmail=is not a valid email address.
diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
index 37274f5306a..31e450a2884 100644
--- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
@@ -217,11 +217,13 @@ public void testInvalidEmail() {
     public void testBoundingBoxValidity() {
         // valid tests
         assertTrue(DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", "-90"));
-        assertTrue(DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", null));
 
         // invalid tests
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("-180", null, "90", null));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox(null, "180", null, "90"));
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", "junk"));
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("45", "40", "90", "0"));
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("360", "0", "90", "-90"));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox(null, null, "90", "-90"));
     }
 }

From 908a624a1b84726a195f961ed60be1a228cb069c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 28 Nov 2023 13:32:20 -0500
Subject: [PATCH 806/922] Checking in the trivial fix for the failing tests.
 Still looking into ways to improve detecting bad facet queries early on (and
 trying to decide if it's out of scope or not). #9635

---
 .../edu/harvard/iq/dataverse/search/SearchIncludeFragment.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index dd9cd78982a..47e82959dd9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -396,7 +396,7 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 }
             }
             
-            if (selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable() && !isFacetsDisabled()) {
+            if (!wasSolrErrorEncountered() && selectedTypesList.size() < 3 && !isSolrTemporarilyUnavailable() && !isFacetsDisabled()) {
                 // If some types are NOT currently selected, we will need to 
                 // run a second search to obtain the numbers of the unselected types:
                 

From 8403a595d7157f06b190796417773feb17e4a733 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 28 Nov 2023 14:25:29 -0500
Subject: [PATCH 807/922] one extra weird edge case (restores the existing
 behavior). #9635

---
 .../iq/dataverse/search/SearchIncludeFragment.java     | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
index 47e82959dd9..41c5ac18d79 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchIncludeFragment.java
@@ -403,14 +403,14 @@ The real issue here (https://github.com/IQSS/dataverse/issues/7304) is caused
                 List<String> filterQueriesFinalSecondPass = new ArrayList<>();
                 filterQueriesFinalSecondPass.addAll(filterQueriesExtended);
                    
-                arr = new String[3 - selectedTypesList.size()];
-                int c = 0; 
+                arr = new String[3];
+                int c = 0;
                 for (String dvObjectType : Arrays.asList("dataverses", "datasets", "files")) {
                     if (!selectedTypesList.contains(dvObjectType)) {
                         arr[c++] = dvObjectType;
                     }
                 }
-                filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ") + ")");
+                filterQueriesFinalSecondPass.add(SearchFields.TYPE + ":(" + combine(arr, " OR ", c) + ")");
                 logger.fine("second pass query: " + queryToPassToSolr);
                 logger.fine("second pass filter query: "+filterQueriesFinalSecondPass.toString());
 
@@ -660,6 +660,10 @@ public void incrementFacets(String name, int incrementNum) {
     // http://stackoverflow.com/questions/1515437/java-function-for-arrays-like-phps-join/1515548#1515548
     String combine(String[] s, String glue) {
         int k = s.length;
+        return combine(s, glue, k);
+    }
+    
+    String combine(String[] s, String glue, int k) {
         if (k == 0) {
             return null;
         }

From aa75382897d52f4264dc463d9adf0d5b450284be Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 28 Nov 2023 14:46:34 -0500
Subject: [PATCH 808/922] address review comments - unused imports/method

---
 .../iq/dataverse/ThumbnailServiceWrapper.java | 57 -------------------
 1 file changed, 57 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
index 8039a161224..ae81a9326c4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java
@@ -5,33 +5,19 @@
  */
 package edu.harvard.iq.dataverse;
 
-import edu.harvard.iq.dataverse.api.Datasets;
-import edu.harvard.iq.dataverse.dataaccess.DataAccess;
-import edu.harvard.iq.dataverse.dataaccess.StorageIO;
-import edu.harvard.iq.dataverse.dataset.DatasetUtil;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 
-import static edu.harvard.iq.dataverse.dataset.DatasetUtil.datasetLogoThumbnail;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
-import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Base64;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.logging.Logger;
 
 import jakarta.ejb.EJB;
 import jakarta.enterprise.context.RequestScoped;
-import jakarta.faces.view.ViewScoped;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
-import org.apache.commons.io.IOUtils;
 
 /**
  *
@@ -59,49 +45,6 @@ public class ThumbnailServiceWrapper implements java.io.Serializable  {
     private Map<Long, DvObject> dvobjectViewMap = new HashMap<>();
     private Map<Long, Boolean> hasThumbMap = new HashMap<>();
 
-    private String getAssignedDatasetImage(Dataset dataset, int size) {
-        if (dataset == null) {
-            return null;
-        }
-
-        DataFile assignedThumbnailFile = dataset.getThumbnailFile();
-
-        if (assignedThumbnailFile != null) {
-            Long assignedThumbnailFileId = assignedThumbnailFile.getId();
-
-            if (this.dvobjectThumbnailsMap.containsKey(assignedThumbnailFileId)) {
-                // Yes, return previous answer
-                //logger.info("using cached result for ... "+assignedThumbnailFileId);
-                if (!"".equals(this.dvobjectThumbnailsMap.get(assignedThumbnailFileId))) {
-                    return this.dvobjectThumbnailsMap.get(assignedThumbnailFileId);
-                }
-                return null;
-            }
-
-            String imageSourceBase64 = ImageThumbConverter.getImageThumbnailAsBase64(assignedThumbnailFile,
-                    size);
-                    //ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
-
-            if (imageSourceBase64 != null) {
-                this.dvobjectThumbnailsMap.put(assignedThumbnailFileId, imageSourceBase64);
-                return imageSourceBase64;
-            }
-
-            // OK - we can't use this "assigned" image, because of permissions, or because 
-            // the thumbnail failed to generate, etc... in this case we'll 
-            // mark this dataset in the lookup map - so that we don't have to
-            // do all these lookups again...
-            this.dvobjectThumbnailsMap.put(assignedThumbnailFileId, "");
-            
-            // TODO: (?)
-            // do we need to cache this datafile object in the view map?
-            // -- L.A., 4.2.2
-        }
-
-        return null;
-
-    }
-
     // it's the responsibility of the user - to make sure the search result
     // passed to this method is of the Datafile type!
     public String getFileCardImageAsBase64Url(SolrSearchResult result) {

From b90a1d793d0a5d97abd9d6e0bafbaf5b1540b215 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 28 Nov 2023 14:47:40 -0500
Subject: [PATCH 809/922] update flyway number

---
 ...thumb-failures.sql => V6.0.0.5__9506-track-thumb-failures.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V5.13.0.2__9506-track-thumb-failures.sql => V6.0.0.5__9506-track-thumb-failures.sql} (100%)

diff --git a/src/main/resources/db/migration/V5.13.0.2__9506-track-thumb-failures.sql b/src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql
similarity index 100%
rename from src/main/resources/db/migration/V5.13.0.2__9506-track-thumb-failures.sql
rename to src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql

From 811743d07b5cf645207539294c7e5c0bb1fb6dea Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Tue, 28 Nov 2023 15:13:27 -0500
Subject: [PATCH 810/922] fixt box validation for missing values

---
 .../iq/dataverse/DatasetFieldValueValidator.java    | 13 +++++++++----
 .../dataverse/DatasetFieldValueValidatorTest.java   |  4 +++-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
index 821497299a9..ea9f6c9c3cc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
@@ -67,12 +67,17 @@ public boolean isValid(DatasetFieldValue value, ConstraintValidatorContext conte
         }
 
         // validate fields that are siblings and depend on each others values
-        if (value.getDatasetField().getParentDatasetFieldCompoundValue() != null) {
+        if (value.getDatasetField().getParentDatasetFieldCompoundValue() != null &&
+                value.getDatasetField().getParentDatasetFieldCompoundValue().getParentDatasetField().getValidationMessage() == null) {
             Optional<String> failureMessage = validateChildConstraints(value.getDatasetField());
             if (failureMessage.isPresent()) {
                 try {
                     context.buildConstraintViolationWithTemplate(dsfType.getParentDatasetFieldType().getDisplayName() +  "  " +
                             BundleUtil.getStringFromBundle(failureMessage.get()) ).addConstraintViolation();
+
+                    // save the failure message in the parent so we don't keep validating the children
+                    value.getDatasetField().getParentDatasetFieldCompoundValue().getParentDatasetField().setValidationMessage(failureMessage.get());
+
                 } catch (NullPointerException npe) {
                 }
                 return false;
@@ -240,15 +245,15 @@ public boolean isValidAuthorIdentifier(String userInput, Pattern pattern) {
         return pattern.matcher(userInput).matches();
     }
 
-    // Validate child fields against each other and return failure message or null if success
+    // Validate child fields against each other and return failure message or Optional.empty() if success
     public Optional<String> validateChildConstraints(DatasetField dsf) {
         final String fieldName = dsf.getDatasetFieldType().getName() != null ? dsf.getDatasetFieldType().getName() : "";
         Optional<String> returnFailureMessage = Optional.empty();
 
         // Validate Child Constraint for Geospatial Bounding Box
         // validate the four points of the box to insure proper layout
-        // only validate on one value for speed. picked northLatitude since it was at the bottom of the UI
-        if (fieldName.equals(DatasetFieldConstant.northLatitude)) {
+        if (fieldName.equals(DatasetFieldConstant.northLatitude) || fieldName.equals(DatasetFieldConstant.westLongitude)
+                || fieldName.equals(DatasetFieldConstant.eastLongitude) || fieldName.equals(DatasetFieldConstant.southLatitude)) {
             final String failureMessage = "dataset.metadata.invalidGeospatialCoordinates";
             DatasetFieldCompoundValue cv = dsf.getParentDatasetFieldCompoundValue();
             List<DatasetField> cdsf = cv.getChildDatasetFields();
diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
index 31e450a2884..3f85acc1f87 100644
--- a/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/DatasetFieldValueValidatorTest.java
@@ -217,6 +217,7 @@ public void testInvalidEmail() {
     public void testBoundingBoxValidity() {
         // valid tests
         assertTrue(DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", "-90"));
+        assertTrue(DatasetFieldValueValidator.validateBoundingBox("0", "0", "0", "0"));
 
         // invalid tests
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("-180", null, "90", null));
@@ -224,6 +225,7 @@ public void testBoundingBoxValidity() {
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("-180", "180", "90", "junk"));
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("45", "40", "90", "0"));
         assertTrue(!DatasetFieldValueValidator.validateBoundingBox("360", "0", "90", "-90"));
-        assertTrue(!DatasetFieldValueValidator.validateBoundingBox(null, null, "90", "-90"));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox("", "", "", ""));
+        assertTrue(!DatasetFieldValueValidator.validateBoundingBox(null, null, null, null));
     }
 }

From 866b5eaa1e7108a483c87b0a2a6b532e1f7ef5c1 Mon Sep 17 00:00:00 2001
From: Juan Pablo Tosca Villanueva <jptosca@gmail.com>
Date: Tue, 28 Nov 2023 16:16:40 -0500
Subject: [PATCH 811/922] Adds -X POST on the docs for validateDatasetJson

---
 doc/sphinx-guides/source/api/native-api.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index bd49f945d4e..1f0c5a62d12 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -549,7 +549,7 @@ The fully expanded example above (without environment variables) looks like this
 
 .. code-block:: bash
 
-  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/validateDatasetJson" -H 'Content-type:application/json' --upload-file dataset.json
+  curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/validateDatasetJson" -H 'Content-type:application/json' --upload-file dataset.json
 
 Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint.
 

From 28cb8663e556a61a8577a24187208738b1cb492c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 28 Nov 2023 19:03:17 -0500
Subject: [PATCH 812/922] basic restassured test added. #8549

---
 .../iq/dataverse/EditDatafilesPage.java       |  6 +--
 .../dataverse/ingest/IngestServiceBean.java   | 27 ++++++-------
 ...l => V6.0.0.5__8549-collection-quotas.sql} |  0
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 38 +++++++++++++++++--
 .../edu/harvard/iq/dataverse/api/UtilIT.java  |  6 +++
 5 files changed, 55 insertions(+), 22 deletions(-)
 rename src/main/resources/db/migration/{V6.0.0.4__8549-collection-quotas.sql => V6.0.0.5__8549-collection-quotas.sql} (100%)

diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
index a3256c9159f..a6f31e24764 100644
--- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java
@@ -1102,11 +1102,7 @@ public String save() {
             }
 
             // Try to save the NEW files permanently: 
-            // ... but first, refresh the session quota specifiction, if defined:
-            if (systemConfig.isStorageQuotasEnforced()) {
-                this.uploadSessionQuota = datafileService.getUploadSessionQuotaLimit(dataset);
-            }
-            List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true, uploadSessionQuota);
+            List<DataFile> filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true); 
             
             // reset the working list of fileMetadatas, as to only include the ones
             // that have been added to the version successfully: 
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 9ef0cf07808..8af9a03f19f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -178,17 +178,13 @@ public class IngestServiceBean {
     // the database by calling the Save command on the dataset and/or version.
     
     // There is way too much going on in this method. :(
+    
+    // @todo: Is this method a good candidate for turning into a dedicated Command? 
     public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
             List<DataFile> newFiles,
             DataFile fileToReplace,
             boolean tabIngest) {
-        return saveAndAddFilesToDataset(version, newFiles, fileToReplace, tabIngest, null);
-    }
-    public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
-            List<DataFile> newFiles,
-            DataFile fileToReplace,
-            boolean tabIngest, 
-            UploadSessionQuotaLimit quota) /*throws FileExceedsMaxSizeException, FileExceedsStorageQuotaException*/ {
+        UploadSessionQuotaLimit uploadSessionQuota = null; 
         List<DataFile> ret = new ArrayList<>();
 
         if (newFiles != null && newFiles.size() > 0) {
@@ -200,6 +196,11 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
             IngestUtil.checkForDuplicateFileNamesFinal(version, newFiles, fileToReplace);
             Dataset dataset = version.getDataset();
 
+            if (systemConfig.isStorageQuotasEnforced()) {
+                // Check if this dataset is subject to any storage quotas:
+                uploadSessionQuota = fileService.getUploadSessionQuotaLimit(dataset);
+            }
+            
             for (DataFile dataFile : newFiles) {
                 boolean unattached = false;
                 boolean savedSuccess = false;
@@ -372,10 +373,10 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                 // the API, this is the single point in the workflow where  
                 // storage quotas are enforced. 
         
-                if (systemConfig.isStorageQuotasEnforced() && quota != null) {
-                    long storageQuotaLimit = quota.getRemainingQuotaInBytes();
+                if (uploadSessionQuota != null) {
+                    long storageQuotaLimit = uploadSessionQuota.getRemainingQuotaInBytes();
                     if (confirmedFileSize > storageQuotaLimit) {
-                        savedSuccess = false; 
+                        savedSuccess = false;
                         logger.warning("file size over quota limit, skipping");
                         // @todo: we need to figure out how to better communicate
                         // this (potentially partial) failure to the user.  
@@ -388,12 +389,12 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                         // risks/accuracy loss?)
                         // This update is performed with a direct native query that 
                         // is supposed to be quite fast. But still. 
-                        logger.info("Incrementing recorded storage use by "+confirmedFileSize+" bytes for dataset "+dataset.getId());
+                        logger.info("Incrementing recorded storage use by " + confirmedFileSize + " bytes for dataset " + dataset.getId());
                         // (@todo: need to consider what happens when this code is called on Create?)
                         storageUseService.incrementStorageSizeRecursively(dataset.getId(), confirmedFileSize);
                         // Adjust quota: 
-                        logger.info("Setting total usage in bytes to "+(quota.getTotalUsageInBytes() + confirmedFileSize));
-                        quota.setTotalUsageInBytes(quota.getTotalUsageInBytes() + confirmedFileSize);
+                        logger.info("Setting total usage in bytes to " + (uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize));
+                        uploadSessionQuota.setTotalUsageInBytes(uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize);
                     }
                 }
 
diff --git a/src/main/resources/db/migration/V6.0.0.4__8549-collection-quotas.sql b/src/main/resources/db/migration/V6.0.0.5__8549-collection-quotas.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.0.0.4__8549-collection-quotas.sql
rename to src/main/resources/db/migration/V6.0.0.5__8549-collection-quotas.sql
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 9ad3c353da8..7e6bb682160 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2400,13 +2400,43 @@ public void testCollectionStorageQuotas() {
         assertEquals("Total quota allocation for this collection: 1,024 bytes", JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
 
         UtilIT.enableSetting(SettingsServiceBean.Key.UseStorageQuotas);
-        
+                
+        String pathToFile306bytes = "src/test/resources/FileRecordJobIT.properties"; 
+        String pathToFile1787bytes = "src/test/resources/datacite.xml";
+
         // Upload a small file: 
         
-        String pathToFileUnder1K = "src/test/resources/FileRecordJobIT.properties"; 
-        String pathToFileOver1K = "src/test/resources/datacite.xml";
+        Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile306bytes, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+        
+        // Check the recorded storage use: 
+        
+        Response checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
+        checkStorageUseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        String expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306 B"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+
+        // Attempt to upload the second file - this should get us over the quota, 
+        // so it should be rejected:
+        
+        uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile1787bytes, Json.createObjectBuilder().build(), apiToken);
+        uploadResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+        // We should get this error message made up from 2 Bundle strings:
+        expectedApiMessage = BundleUtil.getStringFromBundle("file.addreplace.error.ingest_create_file_err");
+        expectedApiMessage = expectedApiMessage + " " + BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded", Arrays.asList("1.7 KB", "718 B"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        
+        // Check Storage Use again - should be unchanged: 
+        
+        checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
+        checkStorageUseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306 B"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+
+        // Disable the quota on the collection; try again:
         
+        // [work in progress]         
         
-        // [To be continued/work in progress]
+        UtilIT.deleteSetting(SettingsServiceBean.Key.UseStorageQuotas);
     }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index fd928b4e5a9..adaa2c07c7f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2996,6 +2996,12 @@ static Response setCollectionQuota(String collectionId, long allocatedSize, Stri
         return response;
     }
     
+    static Response checkCollectionStorageUse(String collectionId, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/dataverses/" + collectionId + "/storage/use");
+    }
+    
     /**
      * Determine the "payload" storage size of a dataverse
      *

From de844575988b9c9c9e406f09755852c6e97b1549 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 07:18:51 -0500
Subject: [PATCH 813/922] missing '/' #8549

---
 src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 6 +++---
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 7e6bb682160..d66f7ca9e6d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2387,17 +2387,17 @@ public void testCollectionStorageQuotas() {
         Response checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
         checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
         // This brand new collection shouldn't have any quota defined yet: 
-        assertEquals("No quota defined for this collection", JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        assertEquals(BundleUtil.getStringFromBundle("dataverse.storage.quota.notdefined"), JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
         
         // Set quota to 1K:
         Response setQuotaResponse = UtilIT.setCollectionQuota(dataverseAlias, 1024, apiToken);
         setQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
-        assertEquals("Storage quota successfully set for the collection", JsonPath.from(setQuotaResponse.body().asString()).getString("data.message"));
+        assertEquals(BundleUtil.getStringFromBundle("dataverse.storage.quota.updated"), JsonPath.from(setQuotaResponse.body().asString()).getString("data.message"));
         
         // Check again:
         checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
         checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
-        assertEquals("Total quota allocation for this collection: 1,024 bytes", JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        assertEquals(BundleUtil.getStringFromBundle("dataverse.storage.quota.allocation", Arrays.asList("1,024")), JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
 
         UtilIT.enableSetting(SettingsServiceBean.Key.UseStorageQuotas);
                 
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index adaa2c07c7f..b8eec5471a1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2992,7 +2992,7 @@ static Response checkCollectionQuota(String collectionId, String apiToken) {
     static Response setCollectionQuota(String collectionId, long allocatedSize, String apiToken) {
         Response response = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
-                .post("/api/dataverses/" + collectionId + "/storage/quota" + allocatedSize);
+                .post("/api/dataverses/" + collectionId + "/storage/quota/" + allocatedSize);
         return response;
     }
     

From 6c67dc2aedc926b4ed0064f313f83abf568ce0ea Mon Sep 17 00:00:00 2001
From: landreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 10:07:54 -0500
Subject: [PATCH 814/922] Update
 src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index 42c47505b70..fca84790866 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -1369,7 +1369,7 @@ public Embargo findEmbargo(Long id) {
     
     /**
      * Checks if the supplied DvObjectContainer (Dataset or Collection; although
-     * only collection-level storage quotas are officially supported as if now)
+     * only collection-level storage quotas are officially supported as of now)
      * has a quota configured, and if not, keeps checking if any of the direct
      * ancestor Collections further up have a configured quota. If it finds one, 
      * it will retrieve the current total content size for that specific ancestor 

From f9f375cfdf4823856c16a68af4d54bb2c491d2ba Mon Sep 17 00:00:00 2001
From: landreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 10:10:23 -0500
Subject: [PATCH 815/922] Update doc/release-notes/8549-collection-quotas.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/release-notes/8549-collection-quotas.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/8549-collection-quotas.md b/doc/release-notes/8549-collection-quotas.md
index e3bf37425ea..29b84213cfb 100644
--- a/doc/release-notes/8549-collection-quotas.md
+++ b/doc/release-notes/8549-collection-quotas.md
@@ -1,3 +1,3 @@
 This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
-Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the deployments. 
+Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the deployment.
 

From f9b58a184de79f27d4ad6454a9e3dc263e4d7ca9 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 12:41:18 -0500
Subject: [PATCH 816/922] test fixes #8549

---
 .../java/edu/harvard/iq/dataverse/api/FilesIT.java     | 10 +++++-----
 src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java |  7 +++++++
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index d66f7ca9e6d..d516e6d4bff 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2413,8 +2413,8 @@ public void testCollectionStorageQuotas() {
         
         Response checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
         checkStorageUseResponse.then().assertThat().statusCode(OK.getStatusCode());
-        String expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306 B"));
-        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        String expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkStorageUseResponse.body().asString()).getString("data.message"));
 
         // Attempt to upload the second file - this should get us over the quota, 
         // so it should be rejected:
@@ -2424,14 +2424,14 @@ public void testCollectionStorageQuotas() {
         // We should get this error message made up from 2 Bundle strings:
         expectedApiMessage = BundleUtil.getStringFromBundle("file.addreplace.error.ingest_create_file_err");
         expectedApiMessage = expectedApiMessage + " " + BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded", Arrays.asList("1.7 KB", "718 B"));
-        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        assertEquals(expectedApiMessage, JsonPath.from(uploadResponse.body().asString()).getString("message"));
         
         // Check Storage Use again - should be unchanged: 
         
         checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
         checkStorageUseResponse.then().assertThat().statusCode(OK.getStatusCode());
-        expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306 B"));
-        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkStorageUseResponse.body().asString()).getString("data.message"));
 
         // Disable the quota on the collection; try again:
         
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index b8eec5471a1..47bdc6ab201 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2996,6 +2996,13 @@ static Response setCollectionQuota(String collectionId, long allocatedSize, Stri
         return response;
     }
     
+    static Response disableCollectionQuota(String collectionId, String apiToken) {
+        Response response = given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .delete("/api/dataverses/" + collectionId + "/storage/quota");
+        return response;
+    }
+    
     static Response checkCollectionStorageUse(String collectionId, String apiToken) {
         return given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)

From cda06a3c941d4d72a535e373e6d4057f5648072b Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 14:19:24 -0500
Subject: [PATCH 817/922] quota tests should be working now #8549

---
 .../dataverse/ingest/IngestServiceBean.java   | 36 ++++++++------
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 47 +++++++++++++++++--
 2 files changed, 66 insertions(+), 17 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 8af9a03f19f..9b3ddd228e9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -372,16 +372,27 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                 // necessary. For other cases, such as the direct uploads via 
                 // the API, this is the single point in the workflow where  
                 // storage quotas are enforced. 
-        
-                if (uploadSessionQuota != null) {
-                    long storageQuotaLimit = uploadSessionQuota.getRemainingQuotaInBytes();
-                    if (confirmedFileSize > storageQuotaLimit) {
-                        savedSuccess = false;
-                        logger.warning("file size over quota limit, skipping");
-                        // @todo: we need to figure out how to better communicate
-                        // this (potentially partial) failure to the user.  
-                        //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(confirmedFileSize), bytesToHumanReadable(storageQuotaLimit)));
-                    } else {
+
+                if (savedSuccess) {
+                    if (uploadSessionQuota != null) {
+                        if (confirmedFileSize > uploadSessionQuota.getRemainingQuotaInBytes()) {
+                            savedSuccess = false;
+                            logger.warning("file size over quota limit, skipping");
+                            // @todo: we need to figure out how to better communicate
+                            // this (potentially partial) failure to the user.  
+                            //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(confirmedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+                        } else {
+
+                            // Adjust quota: 
+                            logger.info("Setting total usage in bytes to " + (uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize));
+                            uploadSessionQuota.setTotalUsageInBytes(uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize);
+                        }
+                    }
+
+                    // ... unless we had to reject the file just now because of 
+                    // the quota limits, increment the storage use record(s):
+                    
+                    if (savedSuccess) {
                         // Update storage use for all the parent dvobjects: 
                         // @todo: Do we want to do this after after *each* file is saved? - there may be 
                         // quite a few files being saved here all at once. We could alternatively
@@ -392,13 +403,10 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                         logger.info("Incrementing recorded storage use by " + confirmedFileSize + " bytes for dataset " + dataset.getId());
                         // (@todo: need to consider what happens when this code is called on Create?)
                         storageUseService.incrementStorageSizeRecursively(dataset.getId(), confirmedFileSize);
-                        // Adjust quota: 
-                        logger.info("Setting total usage in bytes to " + (uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize));
-                        uploadSessionQuota.setTotalUsageInBytes(uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize);
                     }
                 }
 
-                logger.fine("Done! Finished saving new file in permanent storage and adding them to the dataset.");
+                logger.fine("Done! Finished saving new file in permanent storage and adding it to the dataset.");
                 boolean belowLimit = false;
 
                 try {
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index d516e6d4bff..e391e17d8d5 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2384,6 +2384,8 @@ public void testCollectionStorageQuotas() {
         createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
         Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
         
+        System.out.println("dataset id: "+datasetId);
+        
         Response checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
         checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
         // This brand new collection shouldn't have any quota defined yet: 
@@ -2397,8 +2399,11 @@ public void testCollectionStorageQuotas() {
         // Check again:
         checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
         checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
-        assertEquals(BundleUtil.getStringFromBundle("dataverse.storage.quota.allocation", Arrays.asList("1,024")), JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        String expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.quota.allocation", Arrays.asList("1,024"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
 
+        System.out.println(expectedApiMessage);
+        
         UtilIT.enableSetting(SettingsServiceBean.Key.UseStorageQuotas);
                 
         String pathToFile306bytes = "src/test/resources/FileRecordJobIT.properties"; 
@@ -2413,9 +2418,11 @@ public void testCollectionStorageQuotas() {
         
         Response checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
         checkStorageUseResponse.then().assertThat().statusCode(OK.getStatusCode());
-        String expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306"));
+        expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("306"));
         assertEquals(expectedApiMessage, JsonPath.from(checkStorageUseResponse.body().asString()).getString("data.message"));
 
+        System.out.println(expectedApiMessage);
+        
         // Attempt to upload the second file - this should get us over the quota, 
         // so it should be rejected:
         
@@ -2426,6 +2433,8 @@ public void testCollectionStorageQuotas() {
         expectedApiMessage = expectedApiMessage + " " + BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded", Arrays.asList("1.7 KB", "718 B"));
         assertEquals(expectedApiMessage, JsonPath.from(uploadResponse.body().asString()).getString("message"));
         
+        System.out.println(expectedApiMessage);
+        
         // Check Storage Use again - should be unchanged: 
         
         checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
@@ -2435,7 +2444,39 @@ public void testCollectionStorageQuotas() {
 
         // Disable the quota on the collection; try again:
         
-        // [work in progress]         
+        Response disableQuotaResponse = UtilIT.disableCollectionQuota(dataverseAlias, apiToken);
+        disableQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
+        expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.quota.deleted");
+        assertEquals(expectedApiMessage, JsonPath.from(disableQuotaResponse.body().asString()).getString("data.message"));
+
+        // Check again: 
+        
+        checkQuotaResponse = UtilIT.checkCollectionQuota(dataverseAlias, apiToken);
+        checkQuotaResponse.then().assertThat().statusCode(OK.getStatusCode());
+        // ... should say "no quota", again: 
+        assertEquals(BundleUtil.getStringFromBundle("dataverse.storage.quota.notdefined"), JsonPath.from(checkQuotaResponse.body().asString()).getString("data.message"));
+        
+        // And try to upload the larger file again:
+        
+        uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToFile1787bytes, Json.createObjectBuilder().build(), apiToken);
+        // ... should work this time around:
+        uploadResponse.then().assertThat().statusCode(OK.getStatusCode());
+            
+        // Let's confirm that the total storage use has been properly implemented:
+
+        //try {sleep(1000);}catch(InterruptedException ie){}
+        
+        checkStorageUseResponse = UtilIT.checkCollectionStorageUse(dataverseAlias, apiToken);
+        checkStorageUseResponse.then().assertThat().statusCode(OK.getStatusCode());
+        expectedApiMessage = BundleUtil.getStringFromBundle("dataverse.storage.use", Arrays.asList("2,093"));
+        assertEquals(expectedApiMessage, JsonPath.from(checkStorageUseResponse.body().asString()).getString("data.message"));
+
+        System.out.println(expectedApiMessage);
+        
+        // @todo: a test for the storage use hierarchy? - create a couple of 
+        // sub-collections, upload a file into a dataset in the farthest branch 
+        // collection, make sure the usage has been incremented all the way up 
+        // to the root? 
         
         UtilIT.deleteSetting(SettingsServiceBean.Key.UseStorageQuotas);
     }

From 857998d322ba996654082e9f85790c9214bbeae1 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 17:29:56 -0500
Subject: [PATCH 818/922] cosmetic changes to the geospatial block - more
 descriptive labels for the entry form + extra formatting for display. #9547

---
 scripts/api/data/metadatablocks/geospatial.tsv |  8 ++++----
 .../java/propertyFiles/geospatial.properties   | 18 +++++++++---------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/scripts/api/data/metadatablocks/geospatial.tsv b/scripts/api/data/metadatablocks/geospatial.tsv
index a3a8e7efd58..2d86a198a9f 100644
--- a/scripts/api/data/metadatablocks/geospatial.tsv
+++ b/scripts/api/data/metadatablocks/geospatial.tsv
@@ -8,10 +8,10 @@
 	otherGeographicCoverage	Other	Other information on the geographic coverage of the data.		text	4	#VALUE, 	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	geographicCoverage	geospatial
 	geographicUnit	Geographic Unit	Lowest level of geographic aggregation covered by the Dataset, e.g., village, county, region.		text	5		TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		geospatial
 	geographicBoundingBox	Geographic Bounding Box	The fundamental geometric description for any Dataset that models geography is the geographic bounding box. It describes the minimum box, defined by west and east longitudes and north and south latitudes, which includes the largest geographic extent of the  Dataset's geographic coverage. This element is used in the first pass of a coordinate-based search. Inclusion of this element in the codebook is recommended, but is required if the bound polygon box is included. 		none	6		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		geospatial
-	westLongitude	West Longitude	Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= West  Bounding Longitude Value <= 180,0.		text	7		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
-	eastLongitude	East Longitude	Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= East Bounding Longitude Value <= 180,0.		text	8		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
-	northLongitude	North Latitude	Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= North Bounding Latitude Value <= 90,0.		text	9		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
-	southLongitude	South Latitude	Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= South Bounding Latitude Value <= 90,0.		text	10		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	westLongitude	Westernmost (Left) Longitude	Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= West  Bounding Longitude Value <= 180,0.		text	7	#VALUEW	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	eastLongitude	Easternmost (Right) Longitude	Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= East Bounding Longitude Value <= 180,0.		text	8	#VALUEE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	northLongitude	Northernmost (Top) Latitude	Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= North Bounding Latitude Value <= 90,0.		text	9	#VALUEN	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	southLongitude	Southernmost (Bottom) Latitude	Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= South Bounding Latitude Value <= 90,0.		text	10	#VALUES	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
 #controlledVocabulary	DatasetField	Value	identifier	displayOrder											
 	country	Afghanistan		0											
 	country	Albania		1											
diff --git a/src/main/java/propertyFiles/geospatial.properties b/src/main/java/propertyFiles/geospatial.properties
index 04db8d3d05f..86f297c29b9 100644
--- a/src/main/java/propertyFiles/geospatial.properties
+++ b/src/main/java/propertyFiles/geospatial.properties
@@ -8,10 +8,10 @@ datasetfieldtype.city.title=City
 datasetfieldtype.otherGeographicCoverage.title=Other
 datasetfieldtype.geographicUnit.title=Geographic Unit
 datasetfieldtype.geographicBoundingBox.title=Geographic Bounding Box
-datasetfieldtype.westLongitude.title=West Longitude
-datasetfieldtype.eastLongitude.title=East Longitude
-datasetfieldtype.northLongitude.title=North Latitude
-datasetfieldtype.southLongitude.title=South Latitude
+datasetfieldtype.westLongitude.title=Westernmost (Left) Longitude
+datasetfieldtype.eastLongitude.title=Easternmost (Right) Longitude
+datasetfieldtype.northLongitude.title=Northernmost (Top) Latitude
+datasetfieldtype.southLongitude.title=Southernmost (Bottom) Latitude
 datasetfieldtype.geographicCoverage.description=Information on the geographic coverage of the data. Includes the total geographic scope of the data.
 datasetfieldtype.country.description=The country or nation that the Dataset is about.
 datasetfieldtype.state.description=The state or province that the Dataset is about. Use GeoNames for correct spelling and avoid abbreviations.
@@ -89,10 +89,10 @@ controlledvocabulary.country.cook_islands=Cook Islands
 controlledvocabulary.country.costa_rica=Costa Rica
 controlledvocabulary.country.croatia=Croatia
 controlledvocabulary.country.cuba=Cuba
-controlledvocabulary.country.curacao=Curaçao
+controlledvocabulary.country.curacao=Cura\u00e7ao
 controlledvocabulary.country.cyprus=Cyprus
 controlledvocabulary.country.czech_republic=Czech Republic
-controlledvocabulary.country.cote_d'ivoire=Côte d'Ivoire
+controlledvocabulary.country.cote_d'ivoire=C\u00f4te d'Ivoire
 controlledvocabulary.country.denmark=Denmark
 controlledvocabulary.country.djibouti=Djibouti
 controlledvocabulary.country.dominica=Dominica
@@ -216,8 +216,8 @@ controlledvocabulary.country.qatar=Qatar
 controlledvocabulary.country.romania=Romania
 controlledvocabulary.country.russian_federation=Russian Federation
 controlledvocabulary.country.rwanda=Rwanda
-controlledvocabulary.country.reunion=Réunion
-controlledvocabulary.country.saint_barthelemy=Saint Barthélemy
+controlledvocabulary.country.reunion=R\u00e9union
+controlledvocabulary.country.saint_barthelemy=Saint Barth\u00e9lemy
 controlledvocabulary.country.saint_helena,_ascension_and_tristan_da_cunha=Saint Helena, Ascension and Tristan da Cunha
 controlledvocabulary.country.saint_kitts_and_nevis=Saint Kitts and Nevis
 controlledvocabulary.country.saint_lucia=Saint Lucia
@@ -282,4 +282,4 @@ controlledvocabulary.country.western_sahara=Western Sahara
 controlledvocabulary.country.yemen=Yemen
 controlledvocabulary.country.zambia=Zambia
 controlledvocabulary.country.zimbabwe=Zimbabwe
-controlledvocabulary.country.aland_islands=Åland Islands
+controlledvocabulary.country.aland_islands=\u00c5land Islands

From 6391f03767042b37d67a2df42a641492ca892f8f Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Wed, 29 Nov 2023 17:41:27 -0500
Subject: [PATCH 819/922] a short release note stub telling admins to update
 the geospatial block #9547

---
 .../9547-validation-for-geospatial-metadata.md           | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 doc/release-notes/9547-validation-for-geospatial-metadata.md

diff --git a/doc/release-notes/9547-validation-for-geospatial-metadata.md b/doc/release-notes/9547-validation-for-geospatial-metadata.md
new file mode 100644
index 00000000000..a44e1a3732b
--- /dev/null
+++ b/doc/release-notes/9547-validation-for-geospatial-metadata.md
@@ -0,0 +1,9 @@
+Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. (issue 9547). This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed.
+
+For the "upgrade" steps section:
+
+Update Geospatial Metadata Block
+
+- `wget https://github.com/IQSS/dataverse/releases/download/v6.1/geospatial.tsv`
+- `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file @geospatial.tsv`
+

From 38ad2c5ef6454122f4ae61a88231bb0fd626ff6f Mon Sep 17 00:00:00 2001
From: Patrick <943484+iPaat@users.noreply.github.com>
Date: Thu, 30 Nov 2023 10:43:58 +0100
Subject: [PATCH 820/922] Update client-libraries.rst

Add idsc.dataverse to Client Libraries to guides.dataverse.org page.
---
 doc/sphinx-guides/source/api/client-libraries.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index 9612616f127..bd0aa55ba99 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -62,6 +62,8 @@ There are multiple Python modules for interacting with Dataverse APIs.
 
 `Pooch <https://github.com/fatiando/pooch>`_ is a Python library that allows library and application developers to download data. Among other features, it takes care of various protocols, caching in OS-specific locations, checksum verification and adds optional features like progress bars or log messages. Among other popular repositories, Pooch supports Dataverse in the sense that you can reference Dataverse-hosted datasets by just a DOI and Pooch will determine the data repository type, query the Dataverse API for contained files and checksums, giving you an easy interface to download them.
 
+`idsc.dataverse <https://github.com/iza-institute-of-labor-economics/idsc.dataverse>`_ reads metadata and files of datasets from a dataverse dataverse.example1.com and writes them into ~/.idsc/dataverse/api/dataverse.example1.com organized in directories PID_type/prefix/suffix, where PID_type is one of: hdl, doi or ark. It can then ''export'' the local copy of the dataverse from ~/.idsc/dataverse/api/dataverse.example1.com to ~/.idsc/.cache/dataverse.example2.com so that one can upload them to dataverse.example2.com.
+
 R
 -
 

From d9b6cae80e50e17be26687c72d42ac916c85f1ad Mon Sep 17 00:00:00 2001
From: Patrick <943484+iPaat@users.noreply.github.com>
Date: Thu, 30 Nov 2023 10:54:46 +0100
Subject: [PATCH 821/922] Update apps.rst

Add idsc.dataverse to guides.dataverse.org migration scripts list.
---
 doc/sphinx-guides/source/api/apps.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst
index a498c62d3d4..44db666736c 100755
--- a/doc/sphinx-guides/source/api/apps.rst
+++ b/doc/sphinx-guides/source/api/apps.rst
@@ -94,6 +94,13 @@ This series of Python scripts offers a starting point for migrating datasets fro
 
 https://github.com/scholarsportal/dataverse-migration-scripts
 
+idsc.dataverse
+~~~~~~~~~~~~~~
+
+This module can, among others, help you migrate one dataverse to another. (see `migrate.md <https://github.com/iza-institute-of-labor-economics/idsc.dataverse/blob/main/migrate.md>`_)
+
+https://github.com/iza-institute-of-labor-economics/idsc.dataverse
+
 Java
 ----
 

From 41784f39681f58d7af6b4233e7bbbfc0a0206adc Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Thu, 30 Nov 2023 09:20:11 -0500
Subject: [PATCH 822/922] fix for load test where coordinates come in a
 different order

---
 .../iq/dataverse/DatasetFieldValueValidator.java | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
index ea9f6c9c3cc..610bb70ff49 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldValueValidator.java
@@ -255,13 +255,17 @@ public Optional<String> validateChildConstraints(DatasetField dsf) {
         if (fieldName.equals(DatasetFieldConstant.northLatitude) || fieldName.equals(DatasetFieldConstant.westLongitude)
                 || fieldName.equals(DatasetFieldConstant.eastLongitude) || fieldName.equals(DatasetFieldConstant.southLatitude)) {
             final String failureMessage = "dataset.metadata.invalidGeospatialCoordinates";
-            DatasetFieldCompoundValue cv = dsf.getParentDatasetFieldCompoundValue();
-            List<DatasetField> cdsf = cv.getChildDatasetFields();
+
             try {
-                if (cdsf.size() == 4) {
-                    if (!validateBoundingBox(cdsf.get(0).getValue(), cdsf.get(1).getValue(), cdsf.get(2).getValue(), cdsf.get(3).getValue())) {
-                        returnFailureMessage = Optional.of(failureMessage);
-                    }
+                final Map<String, String> coords = new HashMap<>();
+                dsf.getParentDatasetFieldCompoundValue().getChildDatasetFields().forEach(f -> {
+                        coords.put(f.getDatasetFieldType().getName(), f.getValue());
+                });
+                if (!validateBoundingBox(coords.get(DatasetFieldConstant.westLongitude),
+                        coords.get(DatasetFieldConstant.eastLongitude),
+                        coords.get(DatasetFieldConstant.northLatitude),
+                        coords.get(DatasetFieldConstant.southLatitude))) {
+                    returnFailureMessage = Optional.of(failureMessage);
                 }
             } catch (IllegalArgumentException e) { // IllegalArgumentException NumberFormatException
                 returnFailureMessage = Optional.of(failureMessage);

From 2d6efc5ee7f05082bcdcd3c837283b19edb8bec8 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 30 Nov 2023 10:43:05 -0500
Subject: [PATCH 823/922] undoing the formatting for the bbox fields: that
 formatting gets passed to solr! :( #9547

---
 scripts/api/data/metadatablocks/geospatial.tsv | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/api/data/metadatablocks/geospatial.tsv b/scripts/api/data/metadatablocks/geospatial.tsv
index 2d86a198a9f..ce481c1bf84 100644
--- a/scripts/api/data/metadatablocks/geospatial.tsv
+++ b/scripts/api/data/metadatablocks/geospatial.tsv
@@ -8,10 +8,10 @@
 	otherGeographicCoverage	Other	Other information on the geographic coverage of the data.		text	4	#VALUE, 	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	geographicCoverage	geospatial
 	geographicUnit	Geographic Unit	Lowest level of geographic aggregation covered by the Dataset, e.g., village, county, region.		text	5		TRUE	FALSE	TRUE	TRUE	FALSE	FALSE		geospatial
 	geographicBoundingBox	Geographic Bounding Box	The fundamental geometric description for any Dataset that models geography is the geographic bounding box. It describes the minimum box, defined by west and east longitudes and north and south latitudes, which includes the largest geographic extent of the  Dataset's geographic coverage. This element is used in the first pass of a coordinate-based search. Inclusion of this element in the codebook is recommended, but is required if the bound polygon box is included. 		none	6		FALSE	FALSE	TRUE	FALSE	FALSE	FALSE		geospatial
-	westLongitude	Westernmost (Left) Longitude	Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= West  Bounding Longitude Value <= 180,0.		text	7	#VALUEW	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
-	eastLongitude	Easternmost (Right) Longitude	Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= East Bounding Longitude Value <= 180,0.		text	8	#VALUEE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
-	northLongitude	Northernmost (Top) Latitude	Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= North Bounding Latitude Value <= 90,0.		text	9	#VALUEN	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
-	southLongitude	Southernmost (Bottom) Latitude	Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= South Bounding Latitude Value <= 90,0.		text	10	#VALUES	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	westLongitude	Westernmost (Left) Longitude	Westernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= West  Bounding Longitude Value <= 180,0.		text	7		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	eastLongitude	Easternmost (Right) Longitude	Easternmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -180,0 <= East Bounding Longitude Value <= 180,0.		text	8		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	northLongitude	Northernmost (Top) Latitude	Northernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= North Bounding Latitude Value <= 90,0.		text	9		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
+	southLongitude	Southernmost (Bottom) Latitude	Southernmost coordinate delimiting the geographic extent of the Dataset. A valid range of values,  expressed in decimal degrees, is -90,0 <= South Bounding Latitude Value <= 90,0.		text	10		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	geographicBoundingBox	geospatial
 #controlledVocabulary	DatasetField	Value	identifier	displayOrder											
 	country	Afghanistan		0											
 	country	Albania		1											

From c7d4ac053f65496eff2d649d8aee6d59dfbdf578 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 30 Nov 2023 15:11:19 -0500
Subject: [PATCH 824/922] allow no body when not needed

---
 src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 597adb0ced6..5961b428bcb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3856,7 +3856,10 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
         } catch (WrappedResponse wr) {
             return wr.getResponse();
         }
-        JsonObject body = JsonUtil.getJsonObject(jsonBody);
+        JsonObject body = null;
+        if (jsonBody != null) {
+            body = JsonUtil.getJsonObject(jsonBody);
+        }
         Set<String> fileIds = null;
         if (downloadId != null) {
             JsonObject files = globusService.getFilesForDownload(downloadId);
@@ -3864,7 +3867,7 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
                 fileIds = files.keySet();
             }
         } else {
-            if (body.containsKey("fileIds")) {
+            if ((body!=null) && body.containsKey("fileIds")) {
                 Collection<JsonValue> fileVals = body.getJsonArray("fileIds").getValuesAs(JsonValue.class);
                 fileIds = new HashSet<String>(fileVals.size());
                 for (JsonValue fileVal : fileVals) {

From 54d228aca37bc8d53c975617a61dcec12d5f2099 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 30 Nov 2023 15:11:56 -0500
Subject: [PATCH 825/922] fix remote case

---
 .../iq/dataverse/globus/GlobusServiceBean.java        | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index b2ebea14835..d8742fc90d5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -660,12 +660,15 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<FileMeta
     public JsonObject getFilesMap(ArrayList<DataFile> dataFiles, Dataset d) {
         JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
         for (DataFile df : dataFiles) {
-            String[] fileInfo = DataAccess.getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(df.getStorageIdentifier(), d));
-            String driverId = fileInfo[0];
-            String fileLocation= fileInfo[1]; 
-            if(GlobusAccessibleStore.isDataverseManaged(driverId)) {
+            String storageId = df.getStorageIdentifier();
+            String[] parts = DataAccess.getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(storageId, d));
+            String driverId =  parts[0];
+            String fileLocation = parts[1];
+            if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
                 String endpointWithBasePath = GlobusAccessibleStore.getTransferEnpointWithPath(driverId);
                 fileLocation = endpointWithBasePath + "/" + fileLocation;
+            } else {
+                fileLocation = storageId.substring(storageId.lastIndexOf("//") + 2);
             }
             filesBuilder.add(df.getId().toString(), fileLocation);
         }

From d647f810c1e9928b40143019d2581be9411dba95 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 30 Nov 2023 15:38:03 -0500
Subject: [PATCH 826/922] tweak docs, add new APIs to API changelog #9506

---
 doc/sphinx-guides/source/api/changelog.rst  |  4 +++
 doc/sphinx-guides/source/api/native-api.rst | 35 +++++++++------------
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index 1726736e75c..43367890381 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -8,6 +8,10 @@ API Changelog
 6.1
 ---
 
+New
+~~~
+- **/api/admin/clearThumbnailFailureFlag**: See :ref:`thumbnail_reset`.
+
 Changes
 ~~~~~~~
 - **/api/datasets/{id}/versions/{versionId}/citation**: This endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation. See :ref:`get-citation`.
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 882a5a872d1..34605a525ed 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -5360,7 +5360,22 @@ A curl example using an ``ID``
 
 Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`.
 
+.. _thumbnail_reset:
+
+Reset Thumbnail Failure Flags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+If Dataverse attempts to create a thumbnail image for an image or PDF file and the attempt fails, Dataverse will set a flag for the file to avoid repeated attempts to generate the thumbnail.
+For cases where the problem may have been temporary (or fixed in a later Dataverse release), the API calls below can be used to reset this flag for all files or for a given file.
+
+.. code-block:: bash
+
+  export SERVER_URL=https://demo.dataverse.org
+  export FILE_ID=1234
+
+  curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag
+  
+  curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag/$FILE_ID
 
 MyData
 ------
@@ -5399,23 +5414,3 @@ Parameters:
 
 ``per_page`` Number of results returned per page.
 
-
-.. _thumbnail_reset:
-
-Reset Thumbnail Failure Flags
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If Dataverse attempts to create a thumbnail image for an image or pdf file and the attempt fails, Dataverse will set a flag for the file to avoid repeated attempts to generate the thumbnail.
-For cases where the problem may have been temporary (or fixed in a later Dataverse release), two API calls exist to reset this flag for all files or for a given file.
-
-Curl examples
-
-.. code-block:: bash
-
-  export SERVER_URL=http://localhost
-  export fileID=1234
-
-  curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag
-  
-  curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag/$fileID
-

From 2a28e7d86a5b412b4c88176dd27e0e90635383d6 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 30 Nov 2023 15:42:13 -0500
Subject: [PATCH 827/922] fix display with no remote url

---
 src/main/webapp/filesFragment.xhtml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 4c7dc2148b9..fbc48a0e884 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -520,7 +520,9 @@
             </div>
             <div class="label label-info remote-info" jsf:rendered="#{not empty fileMetadata.dataFile.storageIO.remoteStoreName}" 
               title="#{bundle['file.remotelyStored']}">
-              <a jsf:rendered="#{not empty fileMetadata.dataFile.storageIO.remoteStoreUrl}" href="#{fileMetadata.dataFile.storageIO.remoteStoreUrl.toString()}" target="_blank" rel="noopener">#{fileMetadata.dataFile.storageIO.remoteStoreName}</a>
+              <div jsf:rendered="#{not empty fileMetadata.dataFile.storageIO.remoteStoreUrl}">
+                  <a href="#{fileMetadata.dataFile.storageIO.remoteStoreUrl.toString()}" target="_blank" rel="noopener">#{fileMetadata.dataFile.storageIO.remoteStoreName}</a>
+              </div>
               <span jsf:rendered="#{empty fileMetadata.dataFile.storageIO.remoteStoreUrl}">#{fileMetadata.dataFile.storageIO.remoteStoreName}</span>
             </div>
             <!-- END: FILE LEVEL MSGs -->

From 4ad06ba1af38cf84f5b639a605eecaf95a4fe8b1 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 30 Nov 2023 15:54:20 -0500
Subject: [PATCH 828/922] rename previewshavefailed to previewimagefail #9506

This matches previewimageavailable, also in dvobject.

Plus it's clear we aren't talking about shaving. :)
---
 .../edu/harvard/iq/dataverse/DataFileServiceBean.java  |  2 +-
 .../iq/dataverse/DatasetVersionServiceBean.java        |  4 ++--
 src/main/java/edu/harvard/iq/dataverse/DvObject.java   | 10 +++++-----
 src/main/java/edu/harvard/iq/dataverse/api/Admin.java  |  2 +-
 .../iq/dataverse/dataaccess/ImageThumbConverter.java   |  4 ++--
 .../migration/V6.0.0.5__9506-track-thumb-failures.sql  |  2 +-
 6 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
index fae95f12a0c..446c66e5a8b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java
@@ -987,7 +987,7 @@ public boolean isThumbnailAvailable (DataFile file) {
             this.save(file);
             return true;
         }
-        file.setPreviewsHaveFailed(true);
+        file.setPreviewImageFail(true);
         this.save(file);
         return false;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
index d209f7d9e26..1ee517c9831 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java
@@ -825,7 +825,7 @@ public Long getThumbnailByVersionId(Long versionId) {
                         + "AND df.id = o.id "
                         + "AND fm.datasetversion_id = dv.id "
                         + "AND fm.datafile_id = df.id "
-                        + "AND o.previewshavefailed = false "
+                        + "AND o.previewimagefail = false "
                         + "AND df.restricted = false "
                         + "AND df.embargo_id is null "
                         + "AND df.contenttype LIKE 'image/%' "
@@ -859,7 +859,7 @@ public Long getThumbnailByVersionId(Long versionId) {
                         + "AND df.id = o.id "
                         + "AND fm.datasetversion_id = dv.id "
                         + "AND fm.datafile_id = df.id "
-                        + "AND o.previewshavefailed = false "
+                        + "AND o.previewimagefail = false "
                         + "AND df.restricted = false "
                         + "AND df.embargo_id is null "
                         + "AND df.contenttype = 'application/pdf' "
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
index 12f0b63b3a1..c6d4a73bfd9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
@@ -191,14 +191,14 @@ public void setPreviewImageAvailable(boolean status) {
      * real failure in generating the thumbnail. In both cases, we won't want to try
      * again every time the preview/thumbnail is requested for a view.
      */
-    private boolean previewsHaveFailed;
+    private boolean previewImageFail;
 
-    public boolean isPreviewsHaveFailed() {
-        return previewsHaveFailed;
+    public boolean isPreviewImageFail() {
+        return previewImageFail;
     }
 
-    public void setPreviewsHaveFailed(boolean previewsHaveFailed) {
-        this.previewsHaveFailed = previewsHaveFailed;
+    public void setPreviewImageFail(boolean previewImageFail) {
+        this.previewImageFail = previewImageFail;
     }
     
     public Timestamp getModificationTime() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index 2c2f49a0444..b1d31f8d44b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -2429,7 +2429,7 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
     @DELETE
     @Path("/clearThumbnailFailureFlag")
     public Response clearThumbnailFailureFlag() {
-        em.createNativeQuery("UPDATE dvobject SET previewshavefailed = FALSE").executeUpdate();
+        em.createNativeQuery("UPDATE dvobject SET previewimagefail = FALSE").executeUpdate();
         return ok("Thumnail Failure Flags cleared.");
     }
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
index febf659b71a..2de37174a3b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java
@@ -119,9 +119,9 @@ private static boolean isThumbnailAvailable(StorageIO<DataFile> storageIO, int s
     }
 
     private static boolean generateThumbnail(DataFile file, StorageIO<DataFile> storageIO, int size) {
-        logger.log(Level.FINE, (file.isPreviewsHaveFailed() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId());
+        logger.log(Level.FINE, (file.isPreviewImageFail() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId());
         // Don't try to generate if there have been failures:
-        if (!file.isPreviewsHaveFailed()) {
+        if (!file.isPreviewImageFail()) {
             boolean thumbnailGenerated = false;
             if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) {
                 thumbnailGenerated = generateImageThumbnail(storageIO, size);
diff --git a/src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql b/src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql
index 9b12d27db91..156960d2011 100644
--- a/src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql
+++ b/src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql
@@ -1 +1 @@
-ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS previewshavefailed BOOLEAN DEFAULT FALSE;
\ No newline at end of file
+ALTER TABLE dvobject ADD COLUMN IF NOT EXISTS previewimagefail BOOLEAN DEFAULT FALSE;

From 7148158dec36576c33c1cbc96143128769dd938a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 30 Nov 2023 15:56:43 -0500
Subject: [PATCH 829/922] add tests #9506

---
 .../java/edu/harvard/iq/dataverse/api/AdminIT.java | 10 ++++++++++
 .../java/edu/harvard/iq/dataverse/api/UtilIT.java  | 14 +++++++++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
index 0c5de662e8a..c29c8619d8c 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/AdminIT.java
@@ -818,6 +818,16 @@ public void testLoadMetadataBlock_ErrorHandling() {
           message
         );
     }
+    @Test
+    public void testClearThumbnailFailureFlag(){
+        Response nonExistentFile = UtilIT.clearThumbnailFailureFlag(Long.MAX_VALUE);
+        nonExistentFile.prettyPrint();
+        nonExistentFile.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
+        
+        Response clearAllFlags = UtilIT.clearThumbnailFailureFlags();
+        clearAllFlags.prettyPrint();
+        clearAllFlags.then().assertThat().statusCode(OK.getStatusCode());
+    }
     
     @Test
     public void testBannerMessages(){
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 9b264086c27..58edbae18e0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -223,7 +223,19 @@ public static Response validateDataFileHashValue(String fileId,  String apiToken
                 .post("/api/admin/validateDataFileHashValue/" + fileId + "?key=" + apiToken);
         return response;
     }
-    
+
+    public static Response clearThumbnailFailureFlags() {
+        Response response = given()
+                .delete("/api/admin/clearThumbnailFailureFlag");
+        return response;
+    }
+
+    public static Response clearThumbnailFailureFlag(long fileId) {
+        Response response = given()
+                .delete("/api/admin/clearThumbnailFailureFlag/" + fileId);
+        return response;
+    }
+
     private static String getAuthenticatedUserAsJsonString(String persistentUserId, String firstName, String lastName, String authenticationProviderId, String identifier) {
         JsonObjectBuilder builder = Json.createObjectBuilder();
         builder.add("authenticationProviderId", authenticationProviderId);

From 67502ca2326b0536077ad96eb0fe497ca70f37f6 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 30 Nov 2023 15:58:18 -0500
Subject: [PATCH 830/922] fix typos #9506

---
 src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index b1d31f8d44b..1445db81e4c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -2430,7 +2430,7 @@ public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject ur
     @Path("/clearThumbnailFailureFlag")
     public Response clearThumbnailFailureFlag() {
         em.createNativeQuery("UPDATE dvobject SET previewimagefail = FALSE").executeUpdate();
-        return ok("Thumnail Failure Flags cleared.");
+        return ok("Thumbnail Failure Flags cleared.");
     }
     
     @DELETE
@@ -2441,7 +2441,7 @@ public Response clearThumbnailFailureFlagByDatafile(@PathParam("id") String file
             Query deleteQuery = em.createNativeQuery("UPDATE dvobject SET previewshavefailed = FALSE where id = ?");
             deleteQuery.setParameter(1, df.getId());
             deleteQuery.executeUpdate();
-            return ok("Thumnail Failure Flag cleared for file id=: " + df.getId() + ".");
+            return ok("Thumbnail Failure Flag cleared for file id=: " + df.getId() + ".");
         } catch (WrappedResponse r) {
             logger.info("Could not find file with the id: " + fileId);
             return error(Status.BAD_REQUEST, "Could not find file with the id: " + fileId);

From 82f0bc0eef833388b3e20bf48fe8bb46163640ee Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 30 Nov 2023 15:59:05 -0500
Subject: [PATCH 831/922] one more rename to previewimagefail #9506

This should have been part of 4ad06ba1a.
---
 src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
index 1445db81e4c..4cb0521d218 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java
@@ -2438,7 +2438,7 @@ public Response clearThumbnailFailureFlag() {
     public Response clearThumbnailFailureFlagByDatafile(@PathParam("id") String fileId) {
         try {
             DataFile df = findDataFileOrDie(fileId);
-            Query deleteQuery = em.createNativeQuery("UPDATE dvobject SET previewshavefailed = FALSE where id = ?");
+            Query deleteQuery = em.createNativeQuery("UPDATE dvobject SET previewimagefail = FALSE where id = ?");
             deleteQuery.setParameter(1, df.getId());
             deleteQuery.executeUpdate();
             return ok("Thumbnail Failure Flag cleared for file id=: " + df.getId() + ".");

From de2f9a4f6beaad2e34249616dd39748c29e15701 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 30 Nov 2023 16:37:35 -0500
Subject: [PATCH 832/922] popup separate tab for single file download transfer

---
 .../iq/dataverse/FileDownloadServiceBean.java | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
index 7a03f1a35dc..ca3f5b4bded 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java
@@ -20,6 +20,8 @@
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import edu.harvard.iq.dataverse.util.StringUtil;
+import edu.harvard.iq.dataverse.util.URLTokenUtil;
+
 import java.io.IOException;
 import java.sql.Timestamp;
 import java.util.ArrayList;
@@ -310,13 +312,19 @@ private void redirectToCustomZipDownloadService(String customZipServiceUrl, Stri
         }
     }
 
-    private void redirectToDownloadAPI(String downloadType, Long fileId, boolean guestBookRecordAlreadyWritten, Long fileMetadataId) {
-        String fileDownloadUrl = FileUtil.getFileDownloadUrlPath(downloadType, fileId, guestBookRecordAlreadyWritten, fileMetadataId);
-        logger.fine("Redirecting to file download url: " + fileDownloadUrl);
-        try {
-            FacesContext.getCurrentInstance().getExternalContext().redirect(fileDownloadUrl);
-        } catch (IOException ex) {
-            logger.info("Failed to issue a redirect to file download url (" + fileDownloadUrl + "): " + ex);
+    private void redirectToDownloadAPI(String downloadType, Long fileId, boolean guestBookRecordAlreadyWritten,
+            Long fileMetadataId) {
+        String fileDownloadUrl = FileUtil.getFileDownloadUrlPath(downloadType, fileId, guestBookRecordAlreadyWritten,
+                fileMetadataId);
+        if (downloadType.equals("GlobusTransfer")) {
+            PrimeFaces.current().executeScript(URLTokenUtil.getScriptForUrl(fileDownloadUrl));
+        } else {
+            logger.fine("Redirecting to file download url: " + fileDownloadUrl);
+            try {
+                FacesContext.getCurrentInstance().getExternalContext().redirect(fileDownloadUrl);
+            } catch (IOException ex) {
+                logger.info("Failed to issue a redirect to file download url (" + fileDownloadUrl + "): " + ex);
+            }
         }
     }
     

From c82064ace53bcbf5e8b04a24f916fa333f863c9c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Thu, 30 Nov 2023 16:38:17 -0500
Subject: [PATCH 833/922] fix old label in popup required case

---
 src/main/webapp/file-download-button-fragment.xhtml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index 8ef2af40431..318aab1454e 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -80,7 +80,7 @@
                          <f:actionListener binding="#{bean.setTermsGuestbookPopupAction(bundle.download)}"/>
                 <f:setPropertyActionListener target="#{fileMetadataForAction}" value="#{fileMetadata}" />
                 <!-- guest book or terms of use, etc. enabled - open "download popup" first: -->
-                GT: #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
+                <span class="globus-btn ui-icon" title="#{bundle['file.globus.transfer']}"/> #{bundle['file.globus.of']} #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
         </li>
         

From 2644faee02f7001e51d19e474e3ca5b1b1264302 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 30 Nov 2023 18:03:29 -0500
Subject: [PATCH 834/922] Rearranges the code that updates the Storage Use
 records to reflect the size of the saved content. #8549

---
 .../dataverse/ingest/IngestServiceBean.java   | 120 +++++++++++-------
 1 file changed, 76 insertions(+), 44 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 9b3ddd228e9..5efb4c06f48 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -177,7 +177,7 @@ public class IngestServiceBean {
     // It must be called before we attempt to permanently save the files in 
     // the database by calling the Save command on the dataset and/or version.
     
-    // There is way too much going on in this method. :(
+    // !! There is way too much going on in this method. :( !!
     
     // @todo: Is this method a good candidate for turning into a dedicated Command? 
     public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
@@ -195,6 +195,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
             // renamed FOOBAR-1.txt back to FOOBAR.txt...
             IngestUtil.checkForDuplicateFileNamesFinal(version, newFiles, fileToReplace);
             Dataset dataset = version.getDataset();
+            long totalBytesSaved = 0L;
 
             if (systemConfig.isStorageQuotasEnforced()) {
                 // Check if this dataset is subject to any storage quotas:
@@ -205,6 +206,9 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                 boolean unattached = false;
                 boolean savedSuccess = false;
                 if (dataFile.getOwner() == null) {
+                    // is it ever "unattached"? 
+                    // do we ever call this method with dataFile.getOwner() != null? 
+                    // - we really shouldn't be, either. 
                     unattached = true;
                     dataFile.setOwner(dataset);
                 }
@@ -230,31 +234,38 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                         dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation);
 
                         logger.fine("Successfully created a new storageIO object.");
-                        /*
-						 * This commented-out code demonstrates how to copy bytes from a local
-						 * InputStream (or a readChannel) into the writable byte channel of a Dataverse
-						 * DataAccessIO object:
+                        /**
+                         * This commented-out code demonstrates how to copy
+                         * bytes from a local InputStream (or a readChannel)
+                         * into the writable byte channel of a Dataverse
+                         * DataAccessIO object:
                          */
 
- /*
-						 * storageIO.open(DataAccessOption.WRITE_ACCESS);
-						 * 
-						 * writeChannel = storageIO.getWriteChannel(); readChannel = new
-						 * FileInputStream(tempLocationPath.toFile()).getChannel();
-						 * 
-						 * long bytesPerIteration = 16 * 1024; // 16K bytes long start = 0; while (
-						 * start < readChannel.size() ) { readChannel.transferTo(start,
-						 * bytesPerIteration, writeChannel); start += bytesPerIteration; }
+                        /**
+                         * storageIO.open(DataAccessOption.WRITE_ACCESS);
+                         *
+                         * writeChannel = storageIO.getWriteChannel();
+                         * readChannel = new
+                         * FileInputStream(tempLocationPath.toFile()).getChannel();
+                         *
+                         * long bytesPerIteration = 16 * 1024; // 16K bytes long
+                         * start = 0; 
+                         * while ( start < readChannel.size() ) {
+                         *    readChannel.transferTo(start, bytesPerIteration, writeChannel); start += bytesPerIteration;
+                         * }
                          */
 
- /*
-						 * But it's easier to use this convenience method from the DataAccessIO:
-						 * 
-						 * (if the underlying storage method for this file is local filesystem, the
-						 * DataAccessIO will simply copy the file using Files.copy, like this:
-						 * 
-						 * Files.copy(tempLocationPath, storageIO.getFileSystemLocation(),
-						 * StandardCopyOption.REPLACE_EXISTING);
+                        /**
+                         * But it's easier to use this convenience method from
+                         * the DataAccessIO:
+                         *
+                         * (if the underlying storage method for this file is
+                         * local filesystem, the DataAccessIO will simply copy
+                         * the file using Files.copy, like this:
+                         *
+                         * Files.copy(tempLocationPath,
+                         * storageIO.getFileSystemLocation(),
+                         * StandardCopyOption.REPLACE_EXISTING);
                          */
                         dataAccess.savePath(tempLocationPath);
 
@@ -265,7 +276,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                         savedSuccess = true;
                         logger.fine("Success: permanently saved file " + dataFile.getFileMetadata().getLabel());
 
-                        // TODO: reformat this file to remove the many tabs added in cc08330
+                        // TODO: reformat this file to remove the many tabs added in cc08330 - done, I think?
                         extractMetadataNcml(dataFile, tempLocationPath);
 
                     } catch (IOException ioex) {
@@ -375,6 +386,15 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
 
                 if (savedSuccess) {
                     if (uploadSessionQuota != null) {
+                        // It may be worth considering refreshing the quota here, 
+                        // and incrementing the Storage Use record for 
+                        // all the parent objects in real time, as 
+                        // *each* individual file is being saved. I experimented
+                        // with that, but decided against it for performance 
+                        // reasons. But yes, there may be some edge case where 
+                        // parallel multi-file uploads can end up being able 
+                        // to save 2X worth the quota that was available at the 
+                        // beginning of each session. 
                         if (confirmedFileSize > uploadSessionQuota.getRemainingQuotaInBytes()) {
                             savedSuccess = false;
                             logger.warning("file size over quota limit, skipping");
@@ -382,7 +402,6 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                             // this (potentially partial) failure to the user.  
                             //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(confirmedFileSize), bytesToHumanReadable(storageQuotaLimit)));
                         } else {
-
                             // Adjust quota: 
                             logger.info("Setting total usage in bytes to " + (uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize));
                             uploadSessionQuota.setTotalUsageInBytes(uploadSessionQuota.getTotalUsageInBytes() + confirmedFileSize);
@@ -390,19 +409,12 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                     }
 
                     // ... unless we had to reject the file just now because of 
-                    // the quota limits, increment the storage use record(s):
+                    // the quota limits, count the number of bytes saved for the 
+                    // purposes of incrementing the total storage of the parent
+                    // DvObjectContainers:
                     
                     if (savedSuccess) {
-                        // Update storage use for all the parent dvobjects: 
-                        // @todo: Do we want to do this after after *each* file is saved? - there may be 
-                        // quite a few files being saved here all at once. We could alternatively
-                        // perform this update only once, after this loop is completed (are there any
-                        // risks/accuracy loss?)
-                        // This update is performed with a direct native query that 
-                        // is supposed to be quite fast. But still. 
-                        logger.info("Incrementing recorded storage use by " + confirmedFileSize + " bytes for dataset " + dataset.getId());
-                        // (@todo: need to consider what happens when this code is called on Create?)
-                        storageUseService.incrementStorageSizeRecursively(dataset.getId(), confirmedFileSize);
+                        totalBytesSaved += confirmedFileSize; 
                     }
                 }
 
@@ -425,12 +437,14 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                     boolean metadataExtracted = false;
                     boolean metadataExtractedFromNetcdf = false;
                     if (tabIngest && FileUtil.canIngestAsTabular(dataFile)) {
-                        /*
-						 * Note that we don't try to ingest the file right away - instead we mark it as
-						 * "scheduled for ingest", then at the end of the save process it will be queued
-						 * for async. ingest in the background. In the meantime, the file will be
-						 * ingested as a regular, non-tabular file, and appear as such to the user,
-						 * until the ingest job is finished with the Ingest Service.
+                        /**
+                         * Note that we don't try to ingest the file right away
+                         * - instead we mark it as "scheduled for ingest", then
+                         * at the end of the save process it will be queued for
+                         * async. ingest in the background. In the meantime, the
+                         * file will be ingested as a regular, non-tabular file,
+                         * and appear as such to the user, until the ingest job
+                         * is finished with the Ingest Service.
                          */
                         dataFile.SetIngestScheduled();
                     } else if (fileMetadataExtractable(dataFile)) {
@@ -488,6 +502,10 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                     // dataset.getGlobalId());
                     // Make sure the file is attached to the dataset and to the version, if this
                     // hasn't been done yet:
+                    // @todo: but shouldn't we be doing the reverse if we haven't been 
+                    // able to save the file? - disconnect it from the dataset and 
+                    // the version?? - L.A. 2023 
+                    // (that said, is there *ever* a case where dataFile.getOwner() != null ?)
                     if (dataFile.getOwner() == null) {
                         dataFile.setOwner(dataset);
 
@@ -503,8 +521,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                                 DataFileCategory dataFileCategory = dfcIt.next();
 
                                 if (dataFileCategory.getDataset() == null) {
-                                    DataFileCategory newCategory = dataset
-                                            .getCategoryByName(dataFileCategory.getName());
+                                    DataFileCategory newCategory = dataset.getCategoryByName(dataFileCategory.getName());
                                     if (newCategory != null) {
                                         newCategory.addFileMetadata(dataFile.getFileMetadata());
                                         // dataFileCategory = newCategory;
@@ -516,10 +533,25 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                             }
                         }
                     }
+                    
+                    // Hmm. Noticing that the following two things - adding the 
+                    // files to the return list were being 
+                    // done outside of this "if (savedSuccess)" block. I'm pretty
+                    // sure that was wrong. - L.A. 11-30-2023
+                    ret.add(dataFile);
+                    // (unless that is that return value isn't used for anything - ?)
                 }
 
-                ret.add(dataFile);
             }
+            // Update storage use for all the parent dvobjects: 
+            logger.info("Incrementing recorded storage use by " + totalBytesSaved + " bytes for dataset " + dataset.getId());
+            // Q. Need to consider what happens when this code is called on Create?
+            // A. It works on create as well, yes. (the recursive increment
+            // query in the method below does need the parent dataset to 
+            // have the database id. But even if these files have been
+            // uploaded on the Create form, we first save the dataset, and 
+            // then add the files to it. - L.A. 
+            storageUseService.incrementStorageSizeRecursively(dataset.getId(), totalBytesSaved);
         }
 
         return ret;

From dc567848bdfcc9647d0779c01bb57f93ab593d89 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 30 Nov 2023 19:10:43 -0500
Subject: [PATCH 835/922] making the set/delete quota commands superuser-only
 (doh). #8549

---
 .../impl/DeleteCollectionQuotaCommand.java       | 13 ++++++++++++-
 .../command/impl/SetCollectionQuotaCommand.java  | 16 +++++++++++++---
 src/main/java/propertyFiles/Bundle.properties    |  1 +
 .../edu/harvard/iq/dataverse/api/FilesIT.java    |  3 +++
 4 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
index 5fcbad929a9..bdeb9c6e8cb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
@@ -6,20 +6,25 @@
 
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import java.util.logging.Logger;
 
 /**
  *
  * @author landreev
+ *
+ * A superuser-only command:
  */
-@RequiredPermissions(Permission.ManageDataversePermissions)
+@RequiredPermissions({})
 public class DeleteCollectionQuotaCommand  extends AbstractVoidCommand {
 
     private static final Logger logger = Logger.getLogger(DeleteCollectionQuotaCommand.class.getCanonicalName());
@@ -33,6 +38,12 @@ public DeleteCollectionQuotaCommand(DataverseRequest aRequest, Dataverse target)
         
     @Override
     public void executeImpl(CommandContext ctxt) throws CommandException {
+        // first check if  user is a superuser
+        if ( (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser() ) ) {      
+            throw new PermissionException(BundleUtil.getStringFromBundle("dataverse.storage.quota.superusersonly"),
+                this,  null, targetDataverse);                
+        }
+        
         if (targetDataverse == null) {
             throw new IllegalCommandException("", this);
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
index a134cbefdb9..6b0d1bf313a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
@@ -6,6 +6,7 @@
 
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
@@ -13,14 +14,18 @@
 import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
 import edu.harvard.iq.dataverse.storageuse.StorageQuota;
+import edu.harvard.iq.dataverse.util.BundleUtil;
 import java.util.logging.Logger;
 
 /**
  *
  * @author landreev
+ *
+ * A superuser-only command:
  */
-@RequiredPermissions(Permission.ManageDataversePermissions)
+@RequiredPermissions({})
 public class SetCollectionQuotaCommand  extends AbstractVoidCommand {
 
     private static final Logger logger = Logger.getLogger(GetCollectionQuotaCommand.class.getCanonicalName());
@@ -36,13 +41,18 @@ public SetCollectionQuotaCommand(DataverseRequest aRequest, Dataverse target, Lo
         
     @Override
     public void executeImpl(CommandContext ctxt) throws CommandException {
+        // Check if user is a superuser:
+        if ( (!(getUser() instanceof AuthenticatedUser) || !getUser().isSuperuser() ) ) {      
+            throw new PermissionException(BundleUtil.getStringFromBundle("dataverse.storage.quota.superusersonly"),
+                this,  null, dataverse);                
+        }
         
         if (dataverse == null) {
-            throw new IllegalCommandException("", this);
+            throw new IllegalCommandException("Must specify valid collection", this);
         }
         
         if (allocation == null) {
-            throw new IllegalCommandException("", this);
+            throw new IllegalCommandException("Must specify valid allocation in bytes", this);
         }
         
         StorageQuota storageQuota = dataverse.getStorageQuota();
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 748b674a4e1..5033426175c 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -925,6 +925,7 @@ dataverse.storage.quota.allocation=Total quota allocation for this collection: {
 dataverse.storage.quota.notdefined=No quota defined for this collection
 dataverse.storage.quota.updated=Storage quota successfully set for the collection
 dataverse.storage.quota.deleted=Storage quota successfully disabled for the collection
+dataverse.storage.quota.superusersonly=Only superusers can change storage quotas.
 dataverse.storage.use=Total recorded size of the files stored in this collection (user-uploaded files plus the versions in the archival tab-delimited format when applicable): {0} bytes
 dataverse.datasize.ioerror=Fatal IO error while trying to determine the total size of the files stored in the dataverse. Please report this error to the Dataverse administrator.
 dataverse.inherited=(inherited from enclosing Dataverse)
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index e391e17d8d5..915f82a6de2 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -2375,6 +2375,9 @@ public void testCollectionStorageQuotas() {
         Response createUser = UtilIT.createRandomUser();
         createUser.then().assertThat().statusCode(OK.getStatusCode());
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        String username = UtilIT.getUsernameFromResponse(createUser);
+        Response makeSuperUser = UtilIT.makeSuperUser(username);
+        assertEquals(200, makeSuperUser.getStatusCode());
 
         Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
         createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());

From f4eee659021dfaab4dfa9c13e761b7c1875281c5 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Thu, 30 Nov 2023 19:18:15 -0500
Subject: [PATCH 836/922] removing the license template stubs (#8549)

---
 .../engine/command/impl/DeleteCollectionQuotaCommand.java   | 5 -----
 .../engine/command/impl/GetCollectionStorageUseCommand.java | 4 ----
 .../engine/command/impl/SetCollectionQuotaCommand.java      | 6 ------
 .../edu/harvard/iq/dataverse/storageuse/StorageQuota.java   | 4 ----
 .../edu/harvard/iq/dataverse/storageuse/StorageUse.java     | 4 ----
 .../iq/dataverse/storageuse/StorageUseServiceBean.java      | 4 ----
 .../iq/dataverse/storageuse/UploadSessionQuotaLimit.java    | 4 ----
 7 files changed, 31 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
index bdeb9c6e8cb..4015228366b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
@@ -1,11 +1,6 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.engine.command.impl;
 
 import edu.harvard.iq.dataverse.Dataverse;
-import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java
index 40b3128b80d..c30a5a34a81 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetCollectionStorageUseCommand.java
@@ -1,7 +1,3 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.engine.command.impl;
 
 import edu.harvard.iq.dataverse.Dataverse;
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
index 6b0d1bf313a..cf8fb6fd42e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
@@ -1,13 +1,7 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.engine.command.impl;
 
 import edu.harvard.iq.dataverse.Dataverse;
-import edu.harvard.iq.dataverse.authorization.Permission;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
-import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.AbstractVoidCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
index 0cfebe4167a..d00f7041e61 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageQuota.java
@@ -1,7 +1,3 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.storageuse;
 
 import edu.harvard.iq.dataverse.DvObject;
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
index 11a2a8b706c..240fba1037d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
@@ -1,7 +1,3 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.storageuse;
 
 import edu.harvard.iq.dataverse.DvObject;
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
index e92ba43e950..b542a7cd661 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
@@ -1,7 +1,3 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.storageuse;
 
 import edu.harvard.iq.dataverse.DvObjectContainer;
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java
index 06bbe986f70..f7dac52e886 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/UploadSessionQuotaLimit.java
@@ -1,7 +1,3 @@
-/*
- * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
- * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template
- */
 package edu.harvard.iq.dataverse.storageuse;
 
 /**

From f48f3a84a72b212d66a4bae1c1056e31dc8f7e52 Mon Sep 17 00:00:00 2001
From: GPortas <hey@gportas.me>
Date: Fri, 1 Dec 2023 14:50:40 +0000
Subject: [PATCH 837/922] Fixed: DatasetVersionFilesServiceBean order by
 condition for type criteria

---
 .../DatasetVersionFilesServiceBean.java       | 27 +++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
index 78fd896c897..99c3c65e3b8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java
@@ -260,22 +260,27 @@ private Predicate createSearchCriteriaPredicate(DatasetVersion datasetVersion,
         return criteriaBuilder.and(predicates.toArray(new Predicate[]{}));
     }
 
-    private Order createGetFileMetadatasOrder(CriteriaBuilder criteriaBuilder,
-                                              FileOrderCriteria orderCriteria,
-                                              Root<FileMetadata> fileMetadataRoot) {
+    private List<Order> createGetFileMetadatasOrder(CriteriaBuilder criteriaBuilder,
+                                                    FileOrderCriteria orderCriteria,
+                                                    Root<FileMetadata> fileMetadataRoot) {
         Path<Object> label = fileMetadataRoot.get("label");
         Path<Object> dataFile = fileMetadataRoot.get("dataFile");
         Path<Timestamp> publicationDate = dataFile.get("publicationDate");
         Path<Timestamp> createDate = dataFile.get("createDate");
         Expression<Object> orderByLifetimeExpression = criteriaBuilder.selectCase().when(publicationDate.isNotNull(), publicationDate).otherwise(createDate);
-        return switch (orderCriteria) {
-            case NameZA -> criteriaBuilder.desc(label);
-            case Newest -> criteriaBuilder.desc(orderByLifetimeExpression);
-            case Oldest -> criteriaBuilder.asc(orderByLifetimeExpression);
-            case Size -> criteriaBuilder.asc(dataFile.get("filesize"));
-            case Type -> criteriaBuilder.asc(dataFile.get("contentType"));
-            default -> criteriaBuilder.asc(label);
-        };
+        List<Order> orderList = new ArrayList<>();
+        switch (orderCriteria) {
+            case NameZA -> orderList.add(criteriaBuilder.desc(label));
+            case Newest -> orderList.add(criteriaBuilder.desc(orderByLifetimeExpression));
+            case Oldest -> orderList.add(criteriaBuilder.asc(orderByLifetimeExpression));
+            case Size -> orderList.add(criteriaBuilder.asc(dataFile.get("filesize")));
+            case Type -> {
+                orderList.add(criteriaBuilder.asc(dataFile.get("contentType")));
+                orderList.add(criteriaBuilder.asc(label));
+            }
+            default -> orderList.add(criteriaBuilder.asc(label));
+        }
+        return orderList;
     }
 
     private long getOriginalTabularFilesSize(DatasetVersion datasetVersion, FileSearchCriteria searchCriteria) {

From a29942bf4c8c78d7dee34d61fbb73f44b8ec699e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:08:26 -0500
Subject: [PATCH 838/922] add files not accessible by dataverse flag

---
 .../dataaccess/AbstractRemoteOverlayAccessIO.java        | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
index 9de6bf69832..16defc26a4f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
@@ -48,6 +48,11 @@ public abstract class AbstractRemoteOverlayAccessIO<T extends DvObject> extends
     static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
     protected static final String REMOTE_STORE_NAME = "remote-store-name";
     protected static final String REMOTE_STORE_URL = "remote-store-url";
+    
+    // Whether Dataverse can access the file bytes
+    //Currently True for the Globus store when using the S3Connector, and Remote Stores like simple web servers where the URLs resolve to the actual file bits
+    static final String FILES_NOT_ACCESSIBLE_BY_DATAVERSE = "files-not-accessible-by-dataverse";
+
     protected StorageIO<DvObject> baseStore = null;
     protected String path = null;
     protected PoolingHttpClientConnectionManager cm = null;
@@ -329,6 +334,10 @@ protected String getStoragePath() throws IOException {
         logger.fine("fullStoragePath: " + fullStoragePath);
         return fullStoragePath;
     }
+    
+    public static boolean isNotDataverseAccessible(String storeId) {
+        return Boolean.parseBoolean(StorageIO.getConfigParamForDriver(storeId, FILES_NOT_ACCESSIBLE_BY_DATAVERSE));
+    }
 
 
 

From 0d758398b64521e65c0d0d90d963aeb7b01af42d Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:09:03 -0500
Subject: [PATCH 839/922] add Globus store to the normal file upload (as for
 the remote store)

---
 .../java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
index 4a4d3f57f83..a1bcbe49327 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
@@ -263,7 +263,8 @@ public static <T extends DvObject> StorageIO<T> createNewStorageIO(T dvObject, S
         	storageIO = new S3AccessIO<>(dvObject, null, storageDriverId);
         	break;
         case REMOTE:
-            storageIO = createNewStorageIO(dvObject, storageTag, RemoteOverlayAccessIO.getBaseStoreIdFor(storageDriverId)) ;
+        case GLOBUS:
+            storageIO = createNewStorageIO(dvObject, storageTag, AbstractRemoteOverlayAccessIO.getBaseStoreIdFor(storageDriverId)) ;
             break;
         default:
         	logger.warning("Could not find storage driver for: " + storageTag);

From ce8bb6e97ff776777b642ceafb3c1fb7bae6129f Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:10:28 -0500
Subject: [PATCH 840/922] add Globus as a download option in file table header

requires changes to startGlobusTransfer in separate commit
---
 src/main/webapp/dataset.xhtml       |  2 +-
 src/main/webapp/filesFragment.xhtml | 21 ++++++++++++++++-----
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 2f76197e508..0b8983a7770 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -230,7 +230,7 @@
                                                             </ui:fragment>
                                                             <ui:fragment rendered="#{settingsWrapper.globusDownload and settingsWrapper.isGlobusEnabledStorageDriver(DatasetPage.dataset.effectiveStorageDriverId)}">
                                                                 <li>
-                                                                    <h:commandLink styleClass="btn-download" action="#{DatasetPage.startGlobusTransfer()}">
+                                                                    <h:commandLink styleClass="btn-download" action="#{DatasetPage.startGlobusTransfer(true)}">
                                                                         <h:outputText value="Globus Transfer"/>
                                                                     </h:commandLink>
                                                                 </li>
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index fbc48a0e884..3d28e3170f7 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -436,7 +436,7 @@
                 <div jsf:id="downloadButtonBlockNormal" class="btn-group" 
                      jsf:rendered="#{(!(empty DatasetPage.workingVersion.fileMetadatas) 
                                      and DatasetPage.workingVersion.fileMetadatas.size() > 1) and DatasetPage.downloadButtonAvailable
-                                     and !DatasetPage.isVersionHasTabular()}">
+                                     and  !(DatasetPage.isVersionHasTabular()||DatasetPage.isVersionHasGlobus())}">
                     <p:commandLink 
                                    styleClass="btn btn-default btn-download"
                                    disabled="#{false and DatasetPage.lockedFromDownload}"
@@ -448,15 +448,15 @@
                         <span class="glyphicon glyphicon-download-alt"/> #{bundle.download}
                     </p:commandLink>
                 </div>
-                <div jsf:id="downloadButtonBlockTabular" class="btn-group" 
+                <div jsf:id="downloadButtonBlockMultiple" class="btn-group" 
                      jsf:rendered="#{(!(empty DatasetPage.workingVersion.fileMetadatas) 
                                      and DatasetPage.workingVersion.fileMetadatas.size() > 1) and DatasetPage.downloadButtonAvailable
-                                     and DatasetPage.isVersionHasTabular()}">
+                                     and (DatasetPage.isVersionHasTabular()||DatasetPage.isVersionHasGlobus())}">
                     <button type="button" class="btn btn-default btn-download dropdown-toggle" data-toggle="dropdown">  
                         <span class="glyphicon glyphicon-download-alt"/> #{bundle.download} <span class="caret"></span>
                     </button>
                     <ul jsf:id="downloadDropdownOptions" class="dropdown-menu multi-level pull-right text-left">
-                        <li >
+                        <li>
                             <p:commandLink
                                            disabled="#{false and DatasetPage.lockedFromDownload}"
                                            update="@form" oncomplete="showPopup();"
@@ -467,7 +467,7 @@
                                 #{bundle.downloadOriginal}
                             </p:commandLink>
                         </li>
-                        <li> 
+                        <li jsf:rendered="#{DatasetPage.isVersionHasTabular()}">
                             <p:commandLink
                                            disabled="#{false and DatasetPage.lockedFromDownload}"
                                            update="@form" oncomplete="showPopup();"
@@ -478,6 +478,17 @@
                                 #{bundle.downloadArchival}
                             </p:commandLink>
                         </li>
+                        <li  jsf:rendered="#{DatasetPage.isVersionHasGlobus()}"> 
+                            <p:commandLink
+                                           disabled="#{false and DatasetPage.lockedFromDownload}"
+                                           update="@form" oncomplete="showPopup();"
+                                           onclick="if (!testFilesSelected()) return false;" 
+                                           actionListener="#{DatasetPage.startGlobusTransfer(false)}">
+                            <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                            <f:setPropertyActionListener target="#{DatasetPage.fileMetadataForAction}" value="#{null}"/>
+                                #{bundle['file.globus.transfer']}
+                            </p:commandLink>
+                        </li>
                     </ul>
                 </div>
 

From 8e75a3e2f501b3f0e09fbc9cba9041c52f769737 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:11:56 -0500
Subject: [PATCH 841/922] Add logic for Globus transfer of some files

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 112 +++++++++++++-----
 1 file changed, 81 insertions(+), 31 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index a663b8588ad..0b0d0a2e4f5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -11,6 +11,9 @@
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.dataaccess.AbstractRemoteOverlayAccessIO;
+import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter;
 import edu.harvard.iq.dataverse.dataaccess.SwiftAccessIO;
 import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil;
@@ -361,6 +364,7 @@ public void setSelectedHostDataverse(Dataverse selectedHostDataverse) {
      * other boolean. 
      */
     private boolean versionHasTabular = false;
+    private boolean versionHasGlobus = false;
 
     private boolean showIngestSuccess;
     
@@ -2183,10 +2187,19 @@ private String init(boolean initFull) {
         // the total "originals" size of the dataset with direct custom queries; 
         // then we'll be able to drop the lookup hint for DataTable from the 
         // findDeep() method for the version and further speed up the lookup 
-        // a little bit. 
+        // a little bit.
+        boolean globusDownloadEnabled = systemConfig.isGlobusDownload();
         for (FileMetadata fmd : workingVersion.getFileMetadatas()) {
-            if (fmd.getDataFile().isTabularData()) {
+            DataFile df = fmd.getDataFile();
+            if (df.isTabularData()) {
                 versionHasTabular = true;
+            }
+            if(globusDownloadEnabled) {
+                if(GlobusAccessibleStore.isGlobusAccessible(DataAccess.getStorageDriverFromIdentifier(df.getStorageIdentifier()))) {
+                    versionHasGlobus= true;
+                }
+            }
+            if(versionHasTabular &&(!globusDownloadEnabled || versionHasGlobus)) {
                 break;
             }
         }
@@ -2483,6 +2496,10 @@ private DefaultTreeNode createFileTreeNode(FileMetadata fileMetadata, TreeNode p
     public boolean isVersionHasTabular() {
         return versionHasTabular;
     }
+    
+    public boolean isVersionHasGlobus() {
+        return versionHasGlobus;
+    }
 
     public boolean isReadOnly() {
         return readOnly;
@@ -3089,6 +3106,16 @@ public void setSelectedNonDownloadableFiles(List<FileMetadata> selectedNonDownlo
         this.selectedNonDownloadableFiles = selectedNonDownloadableFiles;
     }
 
+    private List<FileMetadata> selectedGlobusTransferableFiles;
+
+    public List<FileMetadata> getSelectedGlobusTransferableFiles() {
+        return selectedGlobusTransferableFiles;
+    }
+
+    public void setSelectedGlobusTransferableFiles(List<FileMetadata> selectedGlobusTransferableFiles) {
+        this.selectedGlobusTransferableFiles = selectedGlobusTransferableFiles;
+    }
+    
     public String getSizeOfDataset() {
         return DatasetUtil.getDownloadSize(workingVersion, false);
     }
@@ -3247,8 +3274,8 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
             }
         }
 
-        //if there are two or more files with a total size
-        //over the zip limit post a "too large" popup
+        //if there are two or more files, with a total size
+        //over the zip limit, post a "too large" popup
         if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) {
             setValidateFilesOutcome("FailSize");
             return false;
@@ -3257,16 +3284,17 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
         // If some of the files were restricted and we had to drop them off the
         // list, and NONE of the files are left on the downloadable list
         // - we show them a "you're out of luck" popup:
-        if (getSelectedDownloadableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty()) {
+        if (getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty()) {
             setValidateFilesOutcome("FailRestricted");
             return false;
         }
 
-        if (!getSelectedDownloadableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty()) {
+        if (!(getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty())
+                && !getSelectedNonDownloadableFiles().isEmpty()) {
             setValidateFilesOutcome("Mixed");
             return true;
         }
-
+        //ToDo - should Mixed not trigger this?
         if (isTermsPopupRequired() || isGuestbookPopupRequiredAtDownload()) {
             setValidateFilesOutcome("GuestbookRequired");
         }
@@ -3302,12 +3330,25 @@ private boolean filterSelectedFiles(){
         setSelectedNonDownloadableFiles(new ArrayList<>());
         setSelectedRestrictedFiles(new ArrayList<>());
         setSelectedUnrestrictedFiles(new ArrayList<>());
+        setSelectedGlobusTransferableFiles(new ArrayList<>());
 
         boolean someFiles = false;
+        boolean globusDownloadEnabled = systemConfig.isGlobusDownload();
         for (FileMetadata fmd : this.selectedFiles){
-            if(this.fileDownloadHelper.canDownloadFile(fmd)){
+            boolean downloadable=this.fileDownloadHelper.canDownloadFile(fmd);
+            
+            boolean globusTransferable = false;
+            if(globusDownloadEnabled) {
+                String driverId = DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier());
+                globusTransferable = GlobusAccessibleStore.isGlobusAccessible(driverId);
+                downloadable = downloadable && !AbstractRemoteOverlayAccessIO.isNotDataverseAccessible(driverId); 
+            }
+            if(downloadable){
                 getSelectedDownloadableFiles().add(fmd);
                 someFiles=true;
+            } else if(globusTransferable) {
+                getSelectedGlobusTransferableFiles().add(fmd);
+                someFiles=true;
             } else {
                 getSelectedNonDownloadableFiles().add(fmd);
             }
@@ -5247,7 +5288,7 @@ public boolean isFileAccessRequestMultiButtonEnabled(){
         }
         return false;
     }
-
+/* These appear to be unused - toDo - delete
     private Boolean downloadButtonAllEnabled = null;
 
     public boolean isDownloadAllButtonEnabled() {
@@ -5276,7 +5317,7 @@ public boolean isDownloadSelectedButtonEnabled(){
         }
         return false;
     }
-
+*/
     public boolean isFileAccessRequestMultiSignUpButtonRequired(){
         if (isSessionUserAuthenticated()){
             return false;
@@ -6277,28 +6318,37 @@ public boolean isHasPublicStore() {
         return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.PublicInstall, StorageIO.isPublicStore(dataset.getEffectiveStorageDriverId()));
     }
     
-    public void startGlobusTransfer() {
-        ApiToken apiToken = null;
-        User user = session.getUser();
-        if (user instanceof AuthenticatedUser) {
-            apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
-        } else if (user instanceof PrivateUrlUser) {
-            PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
-            PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
-            apiToken = new ApiToken();
-            apiToken.setTokenString(privUrl.getToken());
-        }
-        if(fileMetadataForAction!=null) {
-            List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
-            downloadFMList.add(fileMetadataForAction);
-            PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
-        } else {
-            if(getSelectedDownloadableFiles()!=null) {
-                PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, getSelectedDownloadableFiles()));
+    public void startGlobusTransfer(boolean transferAll) {
+        if(transferAll) {
+            this.setSelectedFiles(workingVersion.getFileMetadatas());
+        }
+        boolean validated = validateFilesForDownload(true);
+        if (validated) {
+            ApiToken apiToken = null;
+            User user = session.getUser();
+            if (user instanceof AuthenticatedUser) {
+                apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
+            } else if (user instanceof PrivateUrlUser) {
+                PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
+                PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
+                apiToken = new ApiToken();
+                apiToken.setTokenString(privUrl.getToken());
+            }
+            if (fileMetadataForAction != null) {
+                List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
+                downloadFMList.add(fileMetadataForAction);
+                PrimeFaces.current()
+                        .executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
             } else {
-                //ToDo: For non-public, need the subset that are downloadable by the user
-                //ToDo: For mixed (some in backing store), need the ones in the globus store
-                PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, workingVersion.getFileMetadatas()));
+                if (getSelectedGlobusTransferableFiles() != null) {
+                    PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken,
+                            getSelectedGlobusTransferableFiles()));
+                } else {
+                    // ToDo: For non-public, need the subset that are downloadable by the user
+                    // ToDo: For mixed (some in backing store), need the ones in the globus store
+                    PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken,
+                            workingVersion.getFileMetadatas()));
+                }
             }
         }
     }

From 0e91e6ae59020991513add7e14e09c69641ee71e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:12:20 -0500
Subject: [PATCH 842/922] Convenience method to get store id for a file

---
 src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index df0c3e5a019..776d04e98cc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -1797,5 +1797,11 @@ public static boolean isActivelyEmbargoed(List<FileMetadata> fmdList) {
         }
         return false;
     }
+
+
+    public static String getStorageDriver(DataFile dataFile) {
+        String storageIdentifier = dataFile.getStorageIdentifier();
+        return storageIdentifier.substring(0, storageIdentifier.indexOf(DataAccess.SEPARATOR));
+    }
     
 }

From e5bf3001e39bf8362f9025e85cf3f6626baf15d0 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:14:41 -0500
Subject: [PATCH 843/922] skip inaccessible files when doing validatation

---
 .../command/impl/FinalizeDatasetPublicationCommand.java    | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
index 3da087addd9..89cfc732455 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java
@@ -32,15 +32,13 @@
 import java.util.logging.Logger;
 import edu.harvard.iq.dataverse.GlobalIdServiceBean;
 import edu.harvard.iq.dataverse.batch.util.LoggingUtil;
+import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.engine.command.Command;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import java.util.ArrayList;
 import java.util.concurrent.Future;
 import org.apache.solr.client.solrj.SolrServerException;
 
-import jakarta.ejb.EJB;
-import jakarta.inject.Inject;
-
 
 /**
  *
@@ -350,7 +348,8 @@ private void validateDataFiles(Dataset dataset, CommandContext ctxt) throws Comm
                     // (the decision was made to validate all the files on every
                     // major release; we can revisit the decision if there's any
                     // indication that this makes publishing take significantly longer.
-                    if (maxFileSize == -1 || dataFile.getFilesize() < maxFileSize) {
+                    String driverId = FileUtil.getStorageDriver(dataFile);
+                    if(StorageIO.isDataverseAccessible(driverId) && maxFileSize == -1 || dataFile.getFilesize() < maxFileSize) {
                         FileUtil.validateDataFileChecksum(dataFile);
                     }
                     else {

From 534c99bb0376aeaa25f2d9d54cbe68a8bfb3b6bc Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:15:23 -0500
Subject: [PATCH 844/922] Convenience method re: store supports globus access

---
 .../iq/dataverse/dataaccess/GlobusAccessibleStore.java     | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index aad1dab5eab..d827e40e807 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -58,4 +58,11 @@ public static String getGlobusToken(String storeId) {
         return StorageIO.getConfigParamForDriver(storeId, GLOBUS_TOKEN);
     }
     
+    public static boolean isGlobusAccessible(String storeId) {
+        if(StorageIO.getConfigParamForDriver(storeId, StorageIO.TYPE).equals(DataAccess.GLOBUS)) {
+            return true;
+        }
+        return false;
+    }
+    
 }

From ca1a4f1267b2d52cd38054cca61fbddf6941522b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:16:12 -0500
Subject: [PATCH 845/922] Update to use new isNotDataverseAccessible method in
 getInputStream

---
 .../iq/dataverse/dataaccess/GlobusOverlayAccessIO.java        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 7ec1e2f9e73..3e72fa85d35 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -232,7 +232,9 @@ public long retrieveSizeFromMedia() {
     
     @Override
     public InputStream getInputStream() throws IOException {
-        if(Boolean.parseBoolean(getConfigParam("endpoint-maps-to-base-store"))) {
+        //Currently only supported when using an S3 store with the Globus S3Connector.
+        //ToDo: Support when using a managed Globus endpoint that supports http access
+        if(!AbstractRemoteOverlayAccessIO.isNotDataverseAccessible(endpoint)) {
             return baseStore.getInputStream();
         } else {
             throw new IOException("Not implemented");

From f39fa0715e81aafefd14c92c50171eb436a45491 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:17:03 -0500
Subject: [PATCH 846/922] Convenience method isDataverseAccessible

---
 .../edu/harvard/iq/dataverse/dataaccess/StorageIO.java     | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
index 14fc9254c59..51cdecf64a0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
@@ -57,6 +57,8 @@ public abstract class StorageIO<T extends DvObject> {
     static final String UPLOAD_REDIRECT = "upload-redirect";
     static final String UPLOAD_OUT_OF_BAND = "upload-out-of-band";
     protected static final String DOWNLOAD_REDIRECT = "download-redirect";
+    protected static final String DATAVERSE_INACCESSIBLE = "dataverse-inaccessible";
+
 
     public StorageIO() {
 
@@ -620,6 +622,11 @@ public static boolean isDirectUploadEnabled(String driverId) {
                 || Boolean.parseBoolean(getConfigParamForDriver(driverId, UPLOAD_OUT_OF_BAND));
     }
 
+    //True by default, Stores (e.g. RemoteOverlay, Globus) can set this false to stop attempts to read bytes
+    public static boolean isDataverseAccessible(String driverId) {
+        return (true && !Boolean.parseBoolean(getConfigParamForDriver(driverId, DATAVERSE_INACCESSIBLE)));
+    }
+    
     // Check that storageIdentifier is consistent with store's config
     // False will prevent direct uploads
     static boolean isValidIdentifier(String driverId, String storageId) {

From dc4580232dcfe698010cdc4c20fb77c19482484b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 14:18:05 -0500
Subject: [PATCH 847/922] use correct term (though up and down terms are the
 same)

could also fix for native/http, but not for rsync
---
 src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index e40f55fedd8..3c6992f8ec3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -941,7 +941,7 @@ public boolean isHTTPDownload() {
     }
 
     public boolean isGlobusDownload() {
-        return getMethodAvailable(FileUploadMethods.GLOBUS.toString(), false);
+        return getMethodAvailable(FileDownloadMethods.GLOBUS.toString(), false);
     }
     
     public boolean isGlobusFileDownload() {

From 0bfbb10c355ea1ebc24d2d8bee928c50ca22db41 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Fri, 1 Dec 2023 16:59:38 -0500
Subject: [PATCH 848/922] "manage collections" guide entry. #8549

---
 .../source/admin/collectionquotas.rst           | 17 +++++++++++++++++
 doc/sphinx-guides/source/admin/index.rst        |  1 +
 doc/sphinx-guides/source/api/native-api.rst     | 12 +++++++++++-
 .../iq/dataverse/storageuse/StorageUse.java     |  3 +++
 4 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 doc/sphinx-guides/source/admin/collectionquotas.rst

diff --git a/doc/sphinx-guides/source/admin/collectionquotas.rst b/doc/sphinx-guides/source/admin/collectionquotas.rst
new file mode 100644
index 00000000000..883b6cf0c93
--- /dev/null
+++ b/doc/sphinx-guides/source/admin/collectionquotas.rst
@@ -0,0 +1,17 @@
+Storage Quotas for Collections
+==============================
+
+Please note that this is a new and still experimental feature (as of Dataverse v6.1 release).
+
+Instance admins can now define storage quota limits for specific collections. These limits can be set, changed and/or deleted via the provided APIs (please see the :ref:`collection-storage-quotas` section of the :doc:`/api/native-api` guide). The Read version of the API is available to the individual collection admins (i.e., a collection owner can check on the quota configured for their collection), but only superusers can set, change or disable storage quotas.
+
+Storage quotas are *inherited* by subcollections. In other words, when storage use limit is set for a specific collection, it applies to all the datasets immediately under it and in its sub-collections, unless different quotas are defined there and so on. Each file added to any dataset in that hierarchy counts for the purposes of the quota limit defined for the top collection. A storage quota defined on a child sub-collection overrides whatever quota that may be defined on the parent, or inherited from an ancestor.
+
+For example, a collection ``A`` has the storage quota set to 10GB. It has 3 sub-collections, ``B``, ``C`` and ``D``. Users can keep uploading files into the datasets anywhere in this hierarchy until the combined size of 10GB is reached between them. However, if an admin has reasons to limit one of the sub-collections, ``B`` to 3GB only, that quota can be explicitly set there. This both limits the growth of ``B`` to 3GB, and also *guarantees* that allocation to it. I.e. the contributors to collection ``B`` will be able to keep adding data until the 3GB limit is reached, even after the parent collection ``A`` reaches the combined 10GB limit (at which point ``A`` and all its subcollections except for ``B`` will become read-only).
+
+We do not yet know whether this is going to be a popular, or needed use case - a child collection quota that is different from the quota it inherits from a parent. It is likely that for many instances it will be sufficient to be able to define quotas for collections and have them apply to all the child objects underneath. We will examine the response to this feature and consider making adjustments to this scheme based on it. We are already considering introducing other types of quotas, such as limits by users or specific storage volumes.  
+
+Please note that only the sizes of the main datafiles and the archival tab-delimited format versions, as produced by the ingest process are counted for the purposes of enforcing the limits. Automatically generated "auxiliary" files, such as rescaled image thumbnails and metadata exports for datasets are not.
+
+When quotas are set and enforced, the users will be informed of the remaining storage allocation on the file upload page together with other upload and processing limits.
+
diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst
index ac81aa737a7..633842044b4 100755
--- a/doc/sphinx-guides/source/admin/index.rst
+++ b/doc/sphinx-guides/source/admin/index.rst
@@ -27,6 +27,7 @@ This guide documents the functionality only available to superusers (such as "da
    solr-search-index
    ip-groups
    mail-groups
+   collectionquotas
    monitoring
    reporting-tools-and-queries
    maintenance
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 234d5f37232..7bd334f6a95 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -763,7 +763,8 @@ Collection Storage Quotas
 
   curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota"
 
-Will output the storage quota allocated (in bytes), or a message indicating that the quota is not defined for the collection.
+Will output the storage quota allocated (in bytes), or a message indicating that the quota is not defined for the specific collection. The user identified by the API token must have the ``Manage`` permission on the collection. 
+
 
 To set or change the storage allocation quota for a collection:
 
@@ -771,13 +772,22 @@ To set or change the storage allocation quota for a collection:
 
   curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota/$SIZE_IN_BYTES"
 
+This is API is superuser-only.
+  
+
 To delete a storage quota configured for a collection:
 
 .. code-block:: 
 
   curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota"
 
+This is API is superuser-only.
+
+Use the ``/settings`` API to enable or disable the enforcement of storage quotas that are defined across the instance via the following setting. For example,
+
+.. code-block:: 
 
+   curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:UseStorageQuotas
 
 
 Datasets
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
index 240fba1037d..b777736dc8d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUse.java
@@ -7,10 +7,12 @@
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.Id;
+import jakarta.persistence.Index;
 import jakarta.persistence.JoinColumn;
 import jakarta.persistence.NamedQueries;
 import jakarta.persistence.NamedQuery;
 import jakarta.persistence.OneToOne;
+import jakarta.persistence.Table;
 import java.io.Serializable;
 
 /**
@@ -23,6 +25,7 @@
     @NamedQuery(name = "StorageUse.incrementByteSizeByDvContainerId", query = "UPDATE StorageUse su SET su.sizeInBytes = su.sizeInBytes +:fileSize WHERE su.dvObjectContainer.id =:dvObjectId")
 })
 @Entity
+@Table(indexes = {@Index(columnList="dvobjectcontainer_id")})
 public class StorageUse implements Serializable {
 
     private static final long serialVersionUID = 1L;

From 9af23d23d97413338ce2b800697b19970aca3dd5 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 1 Dec 2023 17:23:09 -0500
Subject: [PATCH 849/922] add mixed/other dialogs for transfer case

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 92 ++++++++++++-------
 src/main/java/propertyFiles/Bundle.properties |  6 +-
 src/main/webapp/dataset.xhtml                 | 48 ++++++++--
 src/main/webapp/filesFragment.xhtml           | 10 +-
 4 files changed, 110 insertions(+), 46 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 0b0d0a2e4f5..47a32987b0b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -365,6 +365,7 @@ public void setSelectedHostDataverse(Dataverse selectedHostDataverse) {
      */
     private boolean versionHasTabular = false;
     private boolean versionHasGlobus = false;
+    private boolean globusTransferRequested = false;
 
     private boolean showIngestSuccess;
     
@@ -3116,6 +3117,16 @@ public void setSelectedGlobusTransferableFiles(List<FileMetadata> selectedGlobus
         this.selectedGlobusTransferableFiles = selectedGlobusTransferableFiles;
     }
     
+    private List<FileMetadata> selectedNonGlobusTransferableFiles;
+
+    public List<FileMetadata> getSelectedNonGlobusTransferableFiles() {
+        return selectedNonGlobusTransferableFiles;
+    }
+
+    public void setSelectedNonGlobusTransferableFiles(List<FileMetadata> selectedNonGlobusTransferableFiles) {
+        this.selectedNonGlobusTransferableFiles = selectedNonGlobusTransferableFiles;
+    }
+    
     public String getSizeOfDataset() {
         return DatasetUtil.getDownloadSize(workingVersion, false);
     }
@@ -3227,7 +3238,7 @@ private void startDownload(boolean downloadOriginal){
         boolean guestbookRequired = isDownloadPopupRequired();
         boolean validate = validateFilesForDownload(downloadOriginal);
         if (validate) {
-            updateGuestbookResponse(guestbookRequired, downloadOriginal);
+            updateGuestbookResponse(guestbookRequired, downloadOriginal, false);
             if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){
                 startMultipleFileDownload();
             }
@@ -3289,8 +3300,9 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
             return false;
         }
 
-        if (!(getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty())
-                && !getSelectedNonDownloadableFiles().isEmpty()) {
+        //Some are selected and there are non-downloadable ones or there are both downloadable and globus transferable files
+        if ((!(getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty())
+                && (!getSelectedNonDownloadableFiles().isEmpty()) || (!getSelectedDownloadableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) {
             setValidateFilesOutcome("Mixed");
             return true;
         }
@@ -3302,7 +3314,7 @@ public boolean validateFilesForDownload(boolean downloadOriginal){
 
     }
 
-    private void updateGuestbookResponse (boolean guestbookRequired, boolean downloadOriginal) {
+    private void updateGuestbookResponse (boolean guestbookRequired, boolean downloadOriginal, boolean isGlobusTransfer) {
         // Note that the GuestbookResponse object may still have information from
         // the last download action performed by the user. For example, it may
         // still have the non-null Datafile in it, if the user has just downloaded
@@ -3310,7 +3322,11 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
         // even if that's not what they are trying to do now.
         // So make sure to reset these values:
         guestbookResponse.setDataFile(null);
-        guestbookResponse.setSelectedFileIds(getSelectedDownloadableFilesIdsString());
+        if(isGlobusTransfer) {
+            guestbookResponse.setSelectedFileIds(getFilesIdsString(getSelectedGlobusTransferableFiles()));
+        } else {
+            guestbookResponse.setSelectedFileIds(getSelectedDownloadableFilesIdsString());
+        }
         if (downloadOriginal) {
             guestbookResponse.setFileFormat("original");
         } else {
@@ -3331,6 +3347,7 @@ private boolean filterSelectedFiles(){
         setSelectedRestrictedFiles(new ArrayList<>());
         setSelectedUnrestrictedFiles(new ArrayList<>());
         setSelectedGlobusTransferableFiles(new ArrayList<>());
+        setSelectedNonGlobusTransferableFiles(new ArrayList<>());
 
         boolean someFiles = false;
         boolean globusDownloadEnabled = systemConfig.isGlobusDownload();
@@ -3346,11 +3363,14 @@ private boolean filterSelectedFiles(){
             if(downloadable){
                 getSelectedDownloadableFiles().add(fmd);
                 someFiles=true;
-            } else if(globusTransferable) {
+            } else {
+                getSelectedNonDownloadableFiles().add(fmd);
+            }
+            if(globusTransferable) {
                 getSelectedGlobusTransferableFiles().add(fmd);
                 someFiles=true;
             } else {
-                getSelectedNonDownloadableFiles().add(fmd);
+                getSelectedNonGlobusTransferableFiles().add(fmd);
             }
             if(fmd.isRestricted()){
                 getSelectedRestrictedFiles().add(fmd); //might be downloadable to user or not
@@ -6318,37 +6338,45 @@ public boolean isHasPublicStore() {
         return settingsWrapper.isTrueForKey(SettingsServiceBean.Key.PublicInstall, StorageIO.isPublicStore(dataset.getEffectiveStorageDriverId()));
     }
     
-    public void startGlobusTransfer(boolean transferAll) {
-        if(transferAll) {
+    public boolean isGlobusTransferRequested() {
+        return globusTransferRequested;
+    }
+    
+    public void startGlobusTransfer(boolean transferAll, boolean popupShown) {
+        if (transferAll) {
             this.setSelectedFiles(workingVersion.getFileMetadatas());
         }
+        boolean guestbookRequired = isDownloadPopupRequired();
+        
         boolean validated = validateFilesForDownload(true);
         if (validated) {
-            ApiToken apiToken = null;
-            User user = session.getUser();
-            if (user instanceof AuthenticatedUser) {
-                apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
-            } else if (user instanceof PrivateUrlUser) {
-                PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
-                PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
-                apiToken = new ApiToken();
-                apiToken.setTokenString(privUrl.getToken());
-            }
-            if (fileMetadataForAction != null) {
-                List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
-                downloadFMList.add(fileMetadataForAction);
-                PrimeFaces.current()
-                        .executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
-            } else {
-                if (getSelectedGlobusTransferableFiles() != null) {
-                    PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken,
-                            getSelectedGlobusTransferableFiles()));
+            globusTransferRequested = true;
+            boolean mixed = "Mixed".equals(getValidateFilesOutcome());
+            // transfer is
+            updateGuestbookResponse(guestbookRequired, true, true);
+            if ((!guestbookRequired && !mixed) || popupShown) {
+                ApiToken apiToken = null;
+                User user = session.getUser();
+                if (user instanceof AuthenticatedUser) {
+                    apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
+                } else if (user instanceof PrivateUrlUser) {
+                    PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
+                    PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
+                    apiToken = new ApiToken();
+                    apiToken.setTokenString(privUrl.getToken());
+                }
+                if (fileMetadataForAction != null) {
+                    List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
+                    downloadFMList.add(fileMetadataForAction);
+                    PrimeFaces.current()
+                            .executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
                 } else {
-                    // ToDo: For non-public, need the subset that are downloadable by the user
-                    // ToDo: For mixed (some in backing store), need the ones in the globus store
-                    PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken,
-                            workingVersion.getFileMetadatas()));
+                    if (getSelectedGlobusTransferableFiles() != null) {
+                        PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken,
+                                getSelectedGlobusTransferableFiles()));
+                    }
                 }
+                globusTransferRequested = false;
             }
         }
     }
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 13e3a675a27..65dd020f27b 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -64,6 +64,7 @@ manager=Manager
 curator=Curator
 explore=Explore
 download=Download
+transfer=Globus Transfer
 downloadOriginal=Original Format
 downloadArchival=Archival Format (.tab)
 deaccession=Deaccession
@@ -1391,6 +1392,7 @@ dataset.accessBtn.header.explore=Explore Options
 dataset.accessBtn.header.configure=Configure Options
 dataset.accessBtn.header.compute=Compute Options
 dataset.accessBtn.download.size=ZIP ({0})
+dataset.accessBtn.transfer.size=({0})
 dataset.accessBtn.too.big=The dataset is too large to download. Please select the files you need from the files table.
 dataset.accessBtn.original.too.big=The dataset is too large to download in the original format. Please select the files you need from the files table.
 dataset.accessBtn.archival.too.big=The dataset is too large to download in the archival format. Please select the files you need from the files table.
@@ -1655,8 +1657,10 @@ dataset.inValidSelectedFilesForDownloadWithEmbargo=Embargoed and/or Restricted F
 dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access.
 dataset.mixedSelectedFilesForDownload=The restricted file(s) selected may not be downloaded because you have not been granted access.
 dataset.mixedSelectedFilesForDownloadWithEmbargo=The embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access.
-
+dataset.mixedSelectedFilesForTransfer=Some file(s) cannot be transferred. (They are restricted, embargoed, or not Globus accessible.)
+dataset.inValidSelectedFilesForTransfer=Ineligible Files Selected
 dataset.downloadUnrestricted=Click Continue to download the files you have access to download.
+dataset.transferUnrestricted=Click Continue to transfer the elligible files.
 
 dataset.requestAccessToRestrictedFiles=You may request access to the restricted file(s) by clicking the Request Access button.
 dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button.
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 0b8983a7770..e50e68ec162 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -178,7 +178,7 @@
                                                                 <!-- NORMAL DOWNLOAD BUTTON (NO TABULAR FILES) -->
                                                                 <li jsf:rendered="#{!DatasetPage.versionHasTabular}">                 
                                                                     <p:commandLink update="@form" actionListener="#{DatasetPage.startDownloadAllOriginal()}" styleClass="btn-download" 
-                                                                                   oncomplete="showPopup();">
+                                                                                   oncomplete="showPopup(false);">
                                                                     #{bundle.download}
                                                                         <h:outputFormat value="#{bundle['dataset.accessBtn.download.size']}">
                                                                             <f:param value="#{DatasetPage.sizeOfDataset}" />
@@ -192,7 +192,7 @@
                                                                     <!-- DOWNLOAD ORIGINAL BUTTON (TABULAR FILES PRESENT) -->
                                                                     <li jsf:rendered="#{ DatasetPage.sizeOfDatasetOrig != '0 B'  and !(DatasetPage.tooLargeToDownloadOriginal) }">
                                                                         <p:commandLink update="@form" actionListener="#{DatasetPage.startDownloadAllOriginal()}" 
-                                                                                       oncomplete="showPopup();"
+                                                                                       oncomplete="showPopup(false);"
                                                                                        styleClass="btn-download">
                                                                             #{bundle.downloadOriginal}
                                                                             <h:outputFormat value="#{bundle['dataset.accessBtn.download.size']}">
@@ -208,7 +208,7 @@
                                                                     </ui:fragment>
                                                                     <!-- DOWNLOAD ARCHIVAL FILES (TABULAR FILES PRESENT) -->
                                                                     <li jsf:rendered="#{!DatasetPage.tooLargeToDownloadArchival}">
-                                                                        <p:commandLink update="@form" oncomplete="showPopup();" 
+                                                                        <p:commandLink update="@form" oncomplete="showPopup(false);" 
                                                                                        actionListener="#{DatasetPage.startDownloadAllArchival()}" styleClass="btn-download">
                                                                                 #{bundle.downloadArchival}
                                                                             <h:outputFormat value="#{bundle['dataset.accessBtn.download.size']}">
@@ -230,9 +230,14 @@
                                                             </ui:fragment>
                                                             <ui:fragment rendered="#{settingsWrapper.globusDownload and settingsWrapper.isGlobusEnabledStorageDriver(DatasetPage.dataset.effectiveStorageDriverId)}">
                                                                 <li>
-                                                                    <h:commandLink styleClass="btn-download" action="#{DatasetPage.startGlobusTransfer(true)}">
-                                                                        <h:outputText value="Globus Transfer"/>
-                                                                    </h:commandLink>
+                                                                    <p:commandLink update="@form" oncomplete="showPopup(true);" 
+                                                                                       actionListener="#{DatasetPage.startGlobusTransfer(true, false)}" styleClass="btn-download">
+                                                                                #{bundle.transfer}
+                                                                        <h:outputFormat value="#{bundle['dataset.accessBtn.transfer.size']}">
+                                                                            <f:param value="#{DatasetPage.sizeOfDataset}" />
+                                                                        </h:outputFormat>
+                                                                        <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
+                                                                        </p:commandLink>
                                                                 </li>
                                                             </ui:fragment>
                                                             
@@ -1095,6 +1100,28 @@
                             </button>
                         </div>
                     </p:dialog>
+                    <p:dialog id="globusTransferMixed" styleClass="smallPopUp" header="#{bundle['dataset.inValidSelectedFilesForTransfer']}" widgetVar="globusTransferMixed" modal="true">
+                        <p class="text-danger"><span class="glyphicon glyphicon-exclamation-sign"/> #{bundle['dataset.mixedSelectedFilesForTransfer']}</p>
+                        <table>
+                            <ui:repeat var="resFile" value="#{DatasetPage.selectedNonGlobusTransferableFiles}" >
+                                <tr>
+                                    <td>#{resFile.label}</td>
+                                </tr>
+                            </ui:repeat>
+                        </table>
+                        <div class="button-block">
+                            <p class="help-block">#{bundle['dataset.transferUnrestricted']}</p>
+                            <p:commandButton styleClass="btn btn-default" value="#{bundle.continue}" onclick="PF('globusTransferMixed').hide()" 
+                                             rendered="#{!DatasetPage.guestbookAndTermsPopupRequired}"
+                                             action="#{DatasetPage.startGlobusTransfer(false, true)}"/>
+                            <p:commandButton styleClass="btn btn-default" value="#{bundle.continue}" onclick="PF('globusTransferMixed').hide();"
+                                             rendered="#{DatasetPage.guestbookAndTermsPopupRequired and !settingsWrapper.rsyncDownload}"
+                                             oncomplete="PF('guestbookAndTermsPopup').show();" />
+                            <button class="btn btn-link" onclick="PF('globusTransferMixed').hide();" type="button">
+                                #{bundle.cancel}
+                            </button>
+                        </div>
+                    </p:dialog>
                     <p:dialog id="deleteConfirmation" styleClass="smallPopUp" header="#{bundle['file.deleteDialog.header']}" widgetVar="deleteConfirmation" modal="true">
                         <p class="text-warning"><span class="glyphicon glyphicon-warning-sign"/> #{bundle['file.deleteDialog.tip']}</p>
                         <div class="button-block">
@@ -1545,6 +1572,7 @@
                             <ui:param name="fileDownloadService" value="#{DatasetPage.fileDownloadService}"/>
                             <ui:param name="termsGuestbookPopupAction" value="#{DatasetPage.termsGuestbookPopupAction}"/>
                             <ui:param name="guestbookPopupRequiredAtDownload" value="#{DatasetPage.guestbookPopupRequiredAtDownload}"/>
+                            <ui:param name="isGlobusTransfer" value="#{DatasetPage.globusTransferRequested}"/>
                         </ui:include>
                     </p:dialog>
                     <!-- Preview Guestbook -->
@@ -1911,10 +1939,14 @@
                         $('button[id$="updateOwnerDataverse"]').trigger('click');
                     }
                     
-                    function showPopup() {
+                    function showPopup(isTransfer) {
                         var outcome = document.getElementById("datasetForm:validateFilesOutcome").value;
                         if (outcome ==='Mixed'){
-                            PF('downloadMixed').show();
+                            if(isTransfer) {
+                               PF('globusTransferMixed').show();
+                            } else {
+                                PF('downloadMixed').show();
+                            }
                         }
                         if (outcome ==='FailEmpty'){
                             PF('selectFilesForDownload').show();
diff --git a/src/main/webapp/filesFragment.xhtml b/src/main/webapp/filesFragment.xhtml
index 3d28e3170f7..58899ab7062 100644
--- a/src/main/webapp/filesFragment.xhtml
+++ b/src/main/webapp/filesFragment.xhtml
@@ -442,7 +442,7 @@
                                    disabled="#{false and DatasetPage.lockedFromDownload}"
                                    onclick="if (!testFilesSelected()) return false;" 
                                    action="#{DatasetPage.startDownloadSelectedOriginal()}"
-                                   update="@form" oncomplete="showPopup();">
+                                   update="@form" oncomplete="showPopup(false);">
                         <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                         <f:setPropertyActionListener target="#{DatasetPage.fileMetadataForAction}" value="#{null}"/>
                         <span class="glyphicon glyphicon-download-alt"/> #{bundle.download}
@@ -459,7 +459,7 @@
                         <li>
                             <p:commandLink
                                            disabled="#{false and DatasetPage.lockedFromDownload}"
-                                           update="@form" oncomplete="showPopup();"
+                                           update="@form" oncomplete="showPopup(false);"
                                            onclick="if (!testFilesSelected()) return false;" 
                                            actionListener="#{DatasetPage.startDownloadSelectedOriginal()}">
                                 <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
@@ -470,7 +470,7 @@
                         <li jsf:rendered="#{DatasetPage.isVersionHasTabular()}">
                             <p:commandLink
                                            disabled="#{false and DatasetPage.lockedFromDownload}"
-                                           update="@form" oncomplete="showPopup();"
+                                           update="@form" oncomplete="showPopup(false);"
                                            onclick="if (!testFilesSelected()) return false;" 
                                            actionListener="#{DatasetPage.startDownloadSelectedArchival()}">
                             <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
@@ -481,9 +481,9 @@
                         <li  jsf:rendered="#{DatasetPage.isVersionHasGlobus()}"> 
                             <p:commandLink
                                            disabled="#{false and DatasetPage.lockedFromDownload}"
-                                           update="@form" oncomplete="showPopup();"
+                                           update="@form" oncomplete="showPopup(true);"
                                            onclick="if (!testFilesSelected()) return false;" 
-                                           actionListener="#{DatasetPage.startGlobusTransfer(false)}">
+                                           actionListener="#{DatasetPage.startGlobusTransfer(false, false)}">
                             <f:actionListener binding="#{DatasetPage.setTermsGuestbookPopupAction(bundle.download)}"/>
                             <f:setPropertyActionListener target="#{DatasetPage.fileMetadataForAction}" value="#{null}"/>
                                 #{bundle['file.globus.transfer']}

From 43105d31ae3d5357e450da3a98cac6886e18a1d3 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Sat, 2 Dec 2023 13:14:28 -0500
Subject: [PATCH 850/922] refactor, handle guestbook at download case

---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 29 ++-----
 .../iq/dataverse/FileDownloadHelper.java      | 36 ++++----
 .../iq/dataverse/GuestbookResponse.java       |  2 +-
 .../dataverse/api/DownloadInstanceWriter.java |  6 +-
 .../dataverse/globus/GlobusServiceBean.java   | 86 ++++++++++++++++---
 .../guestbook-terms-popup-fragment.xhtml      | 13 ++-
 6 files changed, 115 insertions(+), 57 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 47a32987b0b..830e146fa07 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3321,7 +3321,11 @@ private void updateGuestbookResponse (boolean guestbookRequired, boolean downloa
         // a single file; or it may still have the format set to "original" -
         // even if that's not what they are trying to do now.
         // So make sure to reset these values:
-        guestbookResponse.setDataFile(null);
+        if(fileMetadataForAction == null) {
+            guestbookResponse.setDataFile(null);
+        } else {
+            guestbookResponse.setDataFile(fileMetadataForAction.getDataFile());
+        }
         if(isGlobusTransfer) {
             guestbookResponse.setSelectedFileIds(getFilesIdsString(getSelectedGlobusTransferableFiles()));
         } else {
@@ -6355,27 +6359,8 @@ public void startGlobusTransfer(boolean transferAll, boolean popupShown) {
             // transfer is
             updateGuestbookResponse(guestbookRequired, true, true);
             if ((!guestbookRequired && !mixed) || popupShown) {
-                ApiToken apiToken = null;
-                User user = session.getUser();
-                if (user instanceof AuthenticatedUser) {
-                    apiToken = authService.findApiTokenByUser((AuthenticatedUser) user);
-                } else if (user instanceof PrivateUrlUser) {
-                    PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
-                    PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
-                    apiToken = new ApiToken();
-                    apiToken.setTokenString(privUrl.getToken());
-                }
-                if (fileMetadataForAction != null) {
-                    List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
-                    downloadFMList.add(fileMetadataForAction);
-                    PrimeFaces.current()
-                            .executeScript(globusService.getGlobusDownloadScript(dataset, apiToken, downloadFMList));
-                } else {
-                    if (getSelectedGlobusTransferableFiles() != null) {
-                        PrimeFaces.current().executeScript(globusService.getGlobusDownloadScript(dataset, apiToken,
-                                getSelectedGlobusTransferableFiles()));
-                    }
-                }
+                boolean doNotSaveGuestbookResponse = workingVersion.isDraft();
+                globusService.writeGuestbookAndStartTransfer(guestbookResponse, doNotSaveGuestbookResponse);
                 globusTransferRequested = false;
             }
         }
diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
index a6ae7223d9d..4d8100124ec 100644
--- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java
@@ -9,6 +9,7 @@
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.externaltools.ExternalTool;
+import edu.harvard.iq.dataverse.globus.GlobusServiceBean;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
@@ -53,6 +54,9 @@ public class FileDownloadHelper implements java.io.Serializable {
     
     @EJB
     DataFileServiceBean datafileService;
+    
+    @EJB
+    GlobusServiceBean globusService;
 
     private final Map<Long, Boolean> fileDownloadPermissionMap = new HashMap<>(); // { FileMetadata.id : Boolean } 
 
@@ -60,32 +64,32 @@ public FileDownloadHelper() {
         this.filesForRequestAccess = new ArrayList<>();
     }
 
-    // See also @Size(max = 255) in GuestbookResponse
-     private boolean testResponseLength(String value) {
-        return !(value != null && value.length() > 255);
-     }
-
     // This helper method is called from the Download terms/guestbook/etc. popup,
     // when the user clicks the "ok" button. We use it, instead of calling
     // downloadServiceBean directly, in order to differentiate between single
     // file downloads and multiple (batch) downloads - since both use the same
     // terms/etc. popup.
-    public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse) {
+    public void writeGuestbookAndStartDownload(GuestbookResponse guestbookResponse, boolean isGlobusTransfer) {
         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
         guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
          // Note that this method is only ever called from the file-download-popup -
          // meaning we know for the fact that we DO want to save this
          // guestbookResponse permanently in the database.
-        if (guestbookResponse.getSelectedFileIds() != null) {
-            // this is a batch (multiple file) download.
-            // Although here's a chance that this is not really a batch download - i.e.,
-            // there may only be one file on the file list. But the fileDownloadService
-            // method below will check for that, and will redirect to the single download, if
-            // that's the case. -- L.A.
-            fileDownloadService.writeGuestbookAndStartBatchDownload(guestbookResponse);
-        } else if (guestbookResponse.getDataFile() != null) {
-            // this a single file download:
-            fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse);
+        if(isGlobusTransfer) {
+            globusService.writeGuestbookAndStartTransfer(guestbookResponse, true);
+        } else {
+            if (guestbookResponse.getSelectedFileIds() != null) {
+                // this is a batch (multiple file) download.
+                // Although here's a chance that this is not really a batch download - i.e.,
+                // there may only be one file on the file list. But the fileDownloadService
+                // method below will check for that, and will redirect to the single download,
+                // if
+                // that's the case. -- L.A.
+                fileDownloadService.writeGuestbookAndStartBatchDownload(guestbookResponse);
+            } else if (guestbookResponse.getDataFile() != null) {
+                // this a single file download:
+                fileDownloadService.writeGuestbookAndStartFileDownload(guestbookResponse);
+            }
         }
      }
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
index 976f1e084ac..9041ccf887c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
+++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java
@@ -99,7 +99,7 @@ public class GuestbookResponse implements Serializable {
      */
     
     public static final String ACCESS_REQUEST = "AccessRequest";
-    static final String DOWNLOAD = "Download";
+    public static final String DOWNLOAD = "Download";
     static final String SUBSET = "Subset";
     static final String EXPLORE = "Explore";
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
index cc064976982..bcb8799ec9e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
@@ -213,9 +213,9 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
                         if (di.getConversionParam().equals("format")) {
 
                             if ("GlobusTransfer".equals(di.getConversionParamValue())) {
-                                List<FileMetadata> downloadFMList = new ArrayList<FileMetadata>(1);
-                                downloadFMList.add(dataFile.getFileMetadata());
-                                redirect_url_str = globusService.getGlobusAppUrlForDataset(dataFile.getOwner(), false, downloadFMList);
+                                List<DataFile> downloadDFList = new ArrayList<DataFile>(1);
+                                downloadDFList.add(dataFile);
+                                redirect_url_str = globusService.getGlobusAppUrlForDataset(dataFile.getOwner(), false, downloadDFList);
                             }
                         }
                     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index d8742fc90d5..0c991424ce9 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -50,15 +50,19 @@
 import java.util.stream.IntStream;
 
 import org.apache.commons.codec.binary.StringUtils;
+import org.primefaces.PrimeFaces;
 
 import com.google.gson.Gson;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.ApiToken;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
+import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
+import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -73,21 +77,22 @@ public class GlobusServiceBean implements java.io.Serializable {
 
     @EJB
     protected DatasetServiceBean datasetSvc;
-
     @EJB
     protected SettingsServiceBean settingsSvc;
-
     @Inject
     DataverseSession session;
-
     @EJB
     protected AuthenticationServiceBean authSvc;
-
     @EJB
     EjbDataverseEngine commandEngine;
-
     @EJB
     UserNotificationServiceBean userNotificationService;
+    @EJB
+    PrivateUrlServiceBean privateUrlService;
+    @EJB
+    FileDownloadServiceBean fileDownloadService;
+    @EJB
+    DataFileServiceBean dataFileService;
 
     private static final Logger logger = Logger.getLogger(GlobusServiceBean.class.getCanonicalName());
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
@@ -600,7 +605,7 @@ public String getGlobusAppUrlForDataset(Dataset d) {
         return getGlobusAppUrlForDataset(d, true, null);
     }
 
-    public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<FileMetadata> fileMetadataList) {
+    public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<DataFile> dataFiles) {
         String localeCode = session.getLocaleCode();
         ApiToken apiToken = null;
         User user = session.getUser();
@@ -629,10 +634,6 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<FileMeta
                     + "/globusUploadParameters?locale=" + localeCode;
         } else {
             // Download
-            ArrayList<DataFile> dataFiles = new ArrayList<DataFile>(fileMetadataList.size());
-            for (FileMetadata fileMetadata : fileMetadataList) {
-                dataFiles.add(fileMetadata.getDataFile());
-            }
             JsonObject files = getFilesMap(dataFiles, d);
 
             String downloadId = UUID.randomUUID().toString();
@@ -657,7 +658,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<FileMeta
         return finalUrl;
     }
 
-    public JsonObject getFilesMap(ArrayList<DataFile> dataFiles, Dataset d) {
+    public JsonObject getFilesMap(List<DataFile> dataFiles, Dataset d) {
         JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
         for (DataFile df : dataFiles) {
             String storageId = df.getStorageIdentifier();
@@ -675,8 +676,8 @@ public JsonObject getFilesMap(ArrayList<DataFile> dataFiles, Dataset d) {
         return filesBuilder.build();
     }
 
-    public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List<FileMetadata> downloadFMList) {
-        return URLTokenUtil.getScriptForUrl(getGlobusAppUrlForDataset(dataset, false, downloadFMList));
+    public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List<DataFile> downloadDFList) {
+        return URLTokenUtil.getScriptForUrl(getGlobusAppUrlForDataset(dataset, false, downloadDFList));
 
     }
 
@@ -1460,5 +1461,64 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
 
         return endpoint;
     }
+    
+    // This helper method is called from the Download terms/guestbook/etc. popup,
+    // when the user clicks the "ok" button. We use it, instead of calling
+    // downloadServiceBean directly, in order to differentiate between single
+    // file downloads and multiple (batch) downloads - since both use the same
+    // terms/etc. popup.
+    public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, boolean doNotSaveGuestbookResponse) {
+        PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
+        guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
+
+        ApiToken apiToken = null;
+        User user = session.getUser();
+        if (user instanceof AuthenticatedUser) {
+            apiToken = authSvc.findApiTokenByUser((AuthenticatedUser) user);
+        } else if (user instanceof PrivateUrlUser) {
+            PrivateUrlUser privateUrlUser = (PrivateUrlUser) user;
+            PrivateUrl privUrl = privateUrlService.getPrivateUrlFromDatasetId(privateUrlUser.getDatasetId());
+            apiToken = new ApiToken();
+            apiToken.setTokenString(privUrl.getToken());
+        }
+        
+        DataFile df = guestbookResponse.getDataFile();
+        if (df != null) {
+            logger.info("Single datafile case for writeGuestbookAndStartTransfer");
+            List<DataFile> downloadDFList = new ArrayList<DataFile>(1);
+            downloadDFList.add(df);
+            if (!doNotSaveGuestbookResponse) {
+                fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
+            }
+            PrimeFaces.current()
+                    .executeScript(getGlobusDownloadScript(df.getOwner(), apiToken, downloadDFList));
+        } else {
+            //Following FileDownloadServiceBean writeGuestbookAndStartBatchDownload
+            List<String> list = new ArrayList<>(Arrays.asList(guestbookResponse.getSelectedFileIds().split(",")));
+            List<DataFile> selectedFiles = new ArrayList<DataFile>();
+            for (String idAsString : list) {
+                try {
+                    Long fileId = Long.parseLong(idAsString);
+                // If we need to create a GuestBookResponse record, we have to 
+                // look up the DataFile object for this file: 
+                if (!doNotSaveGuestbookResponse) {
+                    df = dataFileService.findCheapAndEasy(fileId);
+                    guestbookResponse.setDataFile(df);
+                    fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
+                    selectedFiles.add(df);
+                }
+                } catch (NumberFormatException nfe) {
+                    logger.warning("A file id passed to the writeGuestbookAndStartTransfer method as a string could not be converted back to Long: " + idAsString);
+                    return;
+                }
+
+            }
+            if (!selectedFiles.isEmpty()) {
+                //Use dataset from one file - files should all be from the same dataset
+                PrimeFaces.current().executeScript(getGlobusDownloadScript(df.getOwner(), apiToken,
+                        selectedFiles));
+            }
+        }
+     }
 
 }
diff --git a/src/main/webapp/guestbook-terms-popup-fragment.xhtml b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
index 34df0c79390..5948047d845 100644
--- a/src/main/webapp/guestbook-terms-popup-fragment.xhtml
+++ b/src/main/webapp/guestbook-terms-popup-fragment.xhtml
@@ -274,8 +274,17 @@
             <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
                              rendered="#{guestbookResponse.fileFormat != 'externalTool' and
                                          guestbookResponse.fileFormat != 'package' and
-                                         popupContext != 'previewTab'}"
-                             actionListener="#{fileDownloadHelper.writeGuestbookAndStartDownload(guestbookResponse)}"
+                                         popupContext != 'previewTab' and not isGlobusTransfer}"
+                             actionListener="#{fileDownloadHelper.writeGuestbookAndStartDownload(guestbookResponse, false)}"
+                             update="guestbookUIFragment">
+                <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
+            </p:commandButton>
+            <!--  REGULAR GLOBUS TRANSFER -->
+            <p:commandButton styleClass="btn btn-default" value="#{bundle.acceptTerms}"
+                             rendered="#{guestbookResponse.fileFormat != 'externalTool' and
+                                         guestbookResponse.fileFormat != 'package' and
+                                         popupContext != 'previewTab' and isGlobusTransfer}"
+                             actionListener="#{fileDownloadHelper.writeGuestbookAndStartDownload(guestbookResponse, true)}"
                              update="guestbookUIFragment">
                 <f:param name="DO_GB_VALIDATION_#{popupContext}" value="true"/>
             </p:commandButton>

From a76158f5903ec73a78b284de90d6491a7e05bfce Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Sat, 2 Dec 2023 13:35:33 -0500
Subject: [PATCH 851/922] suppress download entry when not accessible, refactor

---
 .../edu/harvard/iq/dataverse/DatasetPage.java |   2 +-
 .../harvard/iq/dataverse/SettingsWrapper.java |  22 +++
 .../file-download-button-fragment.xhtml       |   6 +-
 .../dataaccess/GlobusOverlayAccessIOTest.java | 176 ++++++++++++++++++
 4 files changed, 202 insertions(+), 4 deletions(-)
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 830e146fa07..704c1d42228 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -3354,7 +3354,7 @@ private boolean filterSelectedFiles(){
         setSelectedNonGlobusTransferableFiles(new ArrayList<>());
 
         boolean someFiles = false;
-        boolean globusDownloadEnabled = systemConfig.isGlobusDownload();
+        boolean globusDownloadEnabled = settingsWrapper.isGlobusDownload();
         for (FileMetadata fmd : this.selectedFiles){
             boolean downloadable=this.fileDownloadHelper.canDownloadFile(fmd);
             
diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
index 8b7f732d03f..8ab1e87aef2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java
@@ -6,6 +6,8 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.branding.BrandingUtil;
+import edu.harvard.iq.dataverse.dataaccess.AbstractRemoteOverlayAccessIO;
+import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import edu.harvard.iq.dataverse.settings.Setting;
@@ -337,6 +339,26 @@ public boolean isGlobusEnabledStorageDriver(String driverId) {
         return (GlobusAccessibleStore.acceptsGlobusTransfers(driverId) || GlobusAccessibleStore.allowsGlobusReferences(driverId));
     }
     
+    public boolean isDownloadable(FileMetadata fmd) {
+        boolean downloadable=true;
+        if(isGlobusFileDownload()) {
+            String driverId = DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier());
+            
+            downloadable = downloadable && !AbstractRemoteOverlayAccessIO.isNotDataverseAccessible(driverId); 
+        }
+        return downloadable;
+    }
+    
+    public boolean isGlobusTransferable(FileMetadata fmd) {
+        boolean globusTransferable=true;
+        if(isGlobusFileDownload()) {
+            String driverId = DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier());
+            globusTransferable = GlobusAccessibleStore.isGlobusAccessible(driverId);
+        }
+        return globusTransferable;
+    }
+    
+    
     public String getGlobusAppUrl() {
         if (globusAppUrl == null) {
             globusAppUrl = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost");
diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml
index 318aab1454e..9c29fd777a1 100644
--- a/src/main/webapp/file-download-button-fragment.xhtml
+++ b/src/main/webapp/file-download-button-fragment.xhtml
@@ -60,7 +60,7 @@
         
         <!-- GlobusTransfer, orig file only  -->
          <o:importFunctions type="edu.harvard.iq.dataverse.dataaccess.DataAccess" />
-        <li jsf:rendered="#{settingsWrapper.globusFileDownload and settingsWrapper.isGlobusEnabledStorageDriver(DataAccess:getStorageDriverFromIdentifier(fileMetadata.dataFile.storageIdentifier))}">
+        <li jsf:rendered="#{settingsWrapper.globusFileDownload and settingsWrapper.isGlobusTransferable(fileMetadata)}">
             <p:commandLink rendered="#{!(guestbookAndTermsPopupRequired) 
                                       and !(fileMetadata.dataFile.filePackage)}"
                          styleClass="btn-download"
@@ -112,7 +112,7 @@
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
             <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and !(guestbookAndTermsPopupRequired) 
-                                      and !(fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload)}"
+                                      and !(fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload) and settingsWrapper.isDownloadable(fileMetadata)}"
                          styleClass="btn-download"
                          process="@this"
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
@@ -122,7 +122,7 @@
                 #{fileMetadata.dataFile.friendlyType == 'Unknown' ? bundle['file.download.filetype.unknown'] : fileMetadata.dataFile.friendlyType}
             </p:commandLink>
             <p:commandLink rendered="#{!(fileMetadata.dataFile.tabularData) and guestbookAndTermsPopupRequired 
-                                      and !(fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload)}"
+                                      and !(fileMetadata.dataFile.filePackage and systemConfig.HTTPDownload) and settingsWrapper.isDownloadable(fileMetadata)}"
                          styleClass="btn-download"
                          process="@this"
                          disabled="#{(fileMetadata.dataFile.ingestInProgress or lockedFromDownload) ? 'disabled' : ''}" 
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
new file mode 100644
index 00000000000..bf3bcdbfe8e
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2018 Forschungszentrum Jülich GmbH
+ * SPDX-License-Identifier: Apache 2.0
+ */
+package edu.harvard.iq.dataverse.dataaccess;
+
+import edu.harvard.iq.dataverse.DOIServiceBean;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.GlobalIdServiceBean;
+import edu.harvard.iq.dataverse.mocks.MocksFactory;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
+import edu.harvard.iq.dataverse.util.UrlSignerUtil;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import static org.junit.jupiter.api.Assertions.*;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.STRICT_STUBS)
+public class GlobusOverlayAccessIOTest {
+
+    @Mock
+
+    private Dataset dataset;
+    private DataFile datafile;
+    private DataFile localDatafile;
+    private String baseStoreId = "182ad2bda2f-c3508e719076";
+    private String logoPath = "image002.mrc";
+    private String authority = "10.5072";
+    private String identifier = "F2ABCDEF";
+
+    @BeforeEach
+    public void setUp() {
+        System.setProperty("dataverse.files.globus." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH,
+                "d8c42580-6528-4605-9ad8-116a61982644/hdc1");
+        System.setProperty("dataverse.files.globus." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS,
+                "d8c42580-6528-4605-9ad8-116a61982644/hdc1");
+
+        System.setProperty("dataverse.files.globus.globus-token",
+                "YTVlNzFjNzItYWVkYi00Mzg4LTkzNWQtY2NhM2IyODI2MzdmOnErQXRBeWNEMVM3amFWVnB0RlFnRk5zMTc3OFdDa3lGeVZPT3k0RDFpaXM9");
+        System.setProperty("dataverse.files.globus.remote-store-name", "GlobusEndpoint1");
+        System.setProperty("dataverse.files.globus.type", "globus");
+
+        System.setProperty("dataverse.files.globus.managed", "true");
+
+        System.setProperty("dataverse.files.globus.base-store", "file");
+        System.setProperty("dataverse.files.file.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+        System.setProperty("dataverse.files.file.directory", "/tmp/files");
+
+        // System.setProperty("dataverse.files.test.type", "remote");
+        System.setProperty("dataverse.files.globus.label", "globusTest");
+        System.setProperty("dataverse.files.test.base-url", "https://demo.dataverse.org/resources");
+        System.setProperty("dataverse.files.test.base-store", "file");
+        System.setProperty("dataverse.files.test.download-redirect", "true");
+        System.setProperty("dataverse.files.test.remote-store-name", "DemoDataCorp");
+        System.setProperty("dataverse.files.globus.secret-key", "12345"); // Real keys should be much longer, more
+                                                                          // random
+        System.setProperty("dataverse.files.file.type", "file");
+        System.setProperty("dataverse.files.file.label", "default");
+        datafile = MocksFactory.makeDataFile();
+        dataset = MocksFactory.makeDataset();
+        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, authority, identifier, "/",
+                DOIServiceBean.DOI_RESOLVER_URL, null));
+        datafile.setOwner(dataset);
+        datafile.setStorageIdentifier("globus://" + baseStoreId + "//" + logoPath);
+
+        localDatafile = MocksFactory.makeDataFile();
+        localDatafile.setOwner(dataset);
+        localDatafile.setStorageIdentifier("globus://" + baseStoreId);
+    }
+
+    @AfterEach
+    public void tearDown() {
+        System.clearProperty("dataverse.files.test.type");
+        System.clearProperty("dataverse.files.test.label");
+        System.clearProperty("dataverse.files.test.base-url");
+        System.clearProperty("dataverse.files.test.base-store");
+        System.clearProperty("dataverse.files.test.download-redirect");
+        System.clearProperty("dataverse.files.test.label");
+        System.clearProperty("dataverse.files.test.remote-store-name");
+        System.clearProperty("dataverse.files.test.secret-key");
+        System.clearProperty("dataverse.files.file.type");
+        System.clearProperty("dataverse.files.file.label");
+    }
+
+    @Test
+    void testGlobusOverlayFiles() throws IOException {
+        System.clearProperty("dataverse.files.globus.managed");
+        datafile.setStorageIdentifier(
+                "globus://" + baseStoreId + "//d8c42580-6528-4605-9ad8-116a61982644/hdc1/" + logoPath);
+        GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(datafile, null, "globus");
+        System.out.println("Size2 is " + gsio.retrieveSizeFromMedia());
+
+        System.out.println(
+                "NotValid: " + GlobusOverlayAccessIO.isValidIdentifier("globus", "globus://localid//../of/the/hill"));
+        System.out.println(
+                "ValidRemote: " + GlobusOverlayAccessIO.isValidIdentifier("globus", "globus://localid//of/the/hill"));
+        System.setProperty("dataverse.files.globus.managed", "true");
+        datafile.setStorageIdentifier("globus://" + baseStoreId + "//" + logoPath);
+        System.out.println("ValidLocal: "
+                + GlobusOverlayAccessIO.isValidIdentifier("globus", "globus://176e28068b0-1c3f80357c42"));
+
+        // We can read the storageIdentifier and get the driver
+        assertTrue(datafile.getStorageIdentifier()
+                .startsWith(DataAccess.getStorageDriverFromIdentifier(datafile.getStorageIdentifier())));
+        // We can get the driver type from it's ID
+        assertTrue(DataAccess.getDriverType("globus").equals(System.getProperty("dataverse.files.globus.type")));
+        // When we get a StorageIO for the file, it is the right type
+        StorageIO<DataFile> storageIO = DataAccess.getStorageIO(localDatafile);
+        assertTrue(storageIO instanceof GlobusOverlayAccessIO);
+        // When we use it, we can get properties like the remote store name
+        GlobusOverlayAccessIO<DataFile> globusIO = (GlobusOverlayAccessIO<DataFile>) storageIO;
+        assertTrue(
+                globusIO.getRemoteStoreName().equals(System.getProperty("dataverse.files.globus.remote-store-name")));
+
+        String location = globusIO.getStorageLocation();
+        assertEquals("globus:///" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + baseStoreId, location);
+/*
+        // TBD:
+        // And can get a temporary download URL for the main file
+        String signedURL = globusIO.generateTemporaryDownloadUrl(null, null, null);
+        System.out.println(signedURL);
+        // And the URL starts with the right stuff
+        assertTrue(signedURL.startsWith(System.getProperty("dataverse.files.globus." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH) + "/" + logoPath));
+        // And the signature is valid
+        // assertTrue(
+        // UrlSignerUtil.isValidUrl(signedURL, null, null,
+        // System.getProperty("dataverse.files.globus.secret-key")));
+        // And we get an unsigned URL with the right stuff with no key
+        System.clearProperty("dataverse.files.globus.secret-key");
+        String unsignedURL = globusIO.generateTemporaryDownloadUrl(null, null, null);
+        assertTrue(unsignedURL.equals(System.getProperty("dataverse.files.globus.base-url") + "/" + logoPath));
+*/
+        // Once we've opened, we can get the file size (only works if the call to Globus
+        // works)
+        globusIO.open(DataAccessOption.READ_ACCESS);
+        assertTrue(globusIO.getSize() > 0);
+        // If we ask for the path for an aux file, it is correct
+        System.out.println(Paths.get(System.getProperty("dataverse.files.file.directory", "/tmp/files"), authority,
+                identifier, baseStoreId + ".auxobject").toString());
+        System.out.println(globusIO.getAuxObjectAsPath("auxobject").toString());
+        assertTrue(Paths.get(System.getProperty("dataverse.files.file.directory", "/tmp/files"), authority, identifier,
+                baseStoreId + ".auxobject").equals(globusIO.getAuxObjectAsPath("auxobject")));
+        IOException thrown = assertThrows(IOException.class, () -> DataAccess.getStorageIO(localDatafile),
+                "Expected getStorageIO() to throw, but it didn't");
+        // 'test' is the driverId in the IOException messages
+        assertTrue(thrown.getMessage().contains("globus"));
+
+    }
+
+    @Test
+    void testRemoteOverlayIdentifierFormats() throws IOException {
+        System.clearProperty("dataverse.files.globus.managed");
+        datafile.setStorageIdentifier(
+                "globus://" + baseStoreId + "//d8c42580-6528-4605-9ad8-116a61982644/hdc1/" + logoPath);
+        assertTrue(DataAccess.isValidDirectStorageIdentifier(datafile.getStorageIdentifier()));
+        assertFalse(
+                DataAccess.isValidDirectStorageIdentifier(datafile.getStorageIdentifier().replace("globus", "bad")));
+        assertFalse(DataAccess.isValidDirectStorageIdentifier(localDatafile.getStorageIdentifier()));
+        System.setProperty("dataverse.files.globus.managed", "true");
+        assertTrue(DataAccess.isValidDirectStorageIdentifier(localDatafile.getStorageIdentifier()));
+
+    }
+
+}

From 93a586727a3c00069699eb47e5ca5ca3ebbf91cf Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Sat, 2 Dec 2023 17:58:45 -0500
Subject: [PATCH 852/922] remove old testing code

---
 .../dataaccess/GlobusOverlayAccessIO.java     |  46 -----
 .../dataaccess/GlobusOverlayAccessIOTest.java | 176 ------------------
 2 files changed, 222 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 3e72fa85d35..e825af8cf30 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -408,52 +408,6 @@ private static String[] getAllowedEndpoints(String driverId) throws IOException
     }
 
 
-    public static void main(String[] args) {
-        System.out.println("Running the main method");
-        if (args.length > 0) {
-            System.out.printf("List of arguments: {}", Arrays.toString(args));
-        }
-        System.setProperty("dataverse.files.globus.base-url", "globus://d8c42580-6528-4605-9ad8-116a61982644");
-        System.out.println("NotValid: " + isValidIdentifier("globus", "globus://localid//../of/the/hill"));
-        System.out.println("ValidRemote: " + isValidIdentifier("globus", "globus://localid//of/the/hill"));
-        System.setProperty("dataverse.files.globus.managed", "true");
-
-        System.out.println("ValidLocal: " + isValidIdentifier("globus", "globus://176e28068b0-1c3f80357c42"));
-        System.setProperty("dataverse.files.globus.globus-token",
-                "");
-        System.setProperty("dataverse.files.globus.base-store", "file");
-        System.setProperty("dataverse.files.file.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
-        System.setProperty("dataverse.files.file.directory", "/tmp/files");
-        // logger.info(JvmSettings.BASE_URL.lookup("globus"));
-        // logger.info(JvmSettings.GLOBUS_TOKEN.lookup("globus"));
-
-        try {
-            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(
-                    "globus://1234///hdc1/image001.mrc", "globus");
-            logger.info("Size is " + gsio.retrieveSizeFromMedia());
-
-        } catch (IOException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
-        try {
-            DataFile df = new DataFile();
-            Dataset ds = new Dataset();
-            ds.setAuthority("10.5072");
-            ds.setIdentifier("FK21234");
-            df.setOwner(ds);
-            df.setStorageIdentifier("globus://1234///hdc1/image001.mrc");
-            GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(df, null, "globus");
-            logger.info("Size2 is " + gsio.retrieveSizeFromMedia());
-
-        } catch (IOException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
-
-    }
-
-
     @Override
     public void open(DataAccessOption... option) throws IOException {
         // TODO Auto-generated method stub
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
index bf3bcdbfe8e..e69de29bb2d 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
@@ -1,176 +0,0 @@
-/*
- * Copyright 2018 Forschungszentrum Jülich GmbH
- * SPDX-License-Identifier: Apache 2.0
- */
-package edu.harvard.iq.dataverse.dataaccess;
-
-import edu.harvard.iq.dataverse.DOIServiceBean;
-import edu.harvard.iq.dataverse.DataFile;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DvObject;
-import edu.harvard.iq.dataverse.GlobalId;
-import edu.harvard.iq.dataverse.GlobalIdServiceBean;
-import edu.harvard.iq.dataverse.mocks.MocksFactory;
-import edu.harvard.iq.dataverse.settings.JvmSettings;
-import edu.harvard.iq.dataverse.util.UrlSignerUtil;
-
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import static org.junit.jupiter.api.Assertions.*;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.mockito.junit.jupiter.MockitoSettings;
-import org.mockito.quality.Strictness;
-import java.io.IOException;
-import java.nio.file.Paths;
-
-@ExtendWith(MockitoExtension.class)
-@MockitoSettings(strictness = Strictness.STRICT_STUBS)
-public class GlobusOverlayAccessIOTest {
-
-    @Mock
-
-    private Dataset dataset;
-    private DataFile datafile;
-    private DataFile localDatafile;
-    private String baseStoreId = "182ad2bda2f-c3508e719076";
-    private String logoPath = "image002.mrc";
-    private String authority = "10.5072";
-    private String identifier = "F2ABCDEF";
-
-    @BeforeEach
-    public void setUp() {
-        System.setProperty("dataverse.files.globus." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH,
-                "d8c42580-6528-4605-9ad8-116a61982644/hdc1");
-        System.setProperty("dataverse.files.globus." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS,
-                "d8c42580-6528-4605-9ad8-116a61982644/hdc1");
-
-        System.setProperty("dataverse.files.globus.globus-token",
-                "YTVlNzFjNzItYWVkYi00Mzg4LTkzNWQtY2NhM2IyODI2MzdmOnErQXRBeWNEMVM3amFWVnB0RlFnRk5zMTc3OFdDa3lGeVZPT3k0RDFpaXM9");
-        System.setProperty("dataverse.files.globus.remote-store-name", "GlobusEndpoint1");
-        System.setProperty("dataverse.files.globus.type", "globus");
-
-        System.setProperty("dataverse.files.globus.managed", "true");
-
-        System.setProperty("dataverse.files.globus.base-store", "file");
-        System.setProperty("dataverse.files.file.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
-        System.setProperty("dataverse.files.file.directory", "/tmp/files");
-
-        // System.setProperty("dataverse.files.test.type", "remote");
-        System.setProperty("dataverse.files.globus.label", "globusTest");
-        System.setProperty("dataverse.files.test.base-url", "https://demo.dataverse.org/resources");
-        System.setProperty("dataverse.files.test.base-store", "file");
-        System.setProperty("dataverse.files.test.download-redirect", "true");
-        System.setProperty("dataverse.files.test.remote-store-name", "DemoDataCorp");
-        System.setProperty("dataverse.files.globus.secret-key", "12345"); // Real keys should be much longer, more
-                                                                          // random
-        System.setProperty("dataverse.files.file.type", "file");
-        System.setProperty("dataverse.files.file.label", "default");
-        datafile = MocksFactory.makeDataFile();
-        dataset = MocksFactory.makeDataset();
-        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, authority, identifier, "/",
-                DOIServiceBean.DOI_RESOLVER_URL, null));
-        datafile.setOwner(dataset);
-        datafile.setStorageIdentifier("globus://" + baseStoreId + "//" + logoPath);
-
-        localDatafile = MocksFactory.makeDataFile();
-        localDatafile.setOwner(dataset);
-        localDatafile.setStorageIdentifier("globus://" + baseStoreId);
-    }
-
-    @AfterEach
-    public void tearDown() {
-        System.clearProperty("dataverse.files.test.type");
-        System.clearProperty("dataverse.files.test.label");
-        System.clearProperty("dataverse.files.test.base-url");
-        System.clearProperty("dataverse.files.test.base-store");
-        System.clearProperty("dataverse.files.test.download-redirect");
-        System.clearProperty("dataverse.files.test.label");
-        System.clearProperty("dataverse.files.test.remote-store-name");
-        System.clearProperty("dataverse.files.test.secret-key");
-        System.clearProperty("dataverse.files.file.type");
-        System.clearProperty("dataverse.files.file.label");
-    }
-
-    @Test
-    void testGlobusOverlayFiles() throws IOException {
-        System.clearProperty("dataverse.files.globus.managed");
-        datafile.setStorageIdentifier(
-                "globus://" + baseStoreId + "//d8c42580-6528-4605-9ad8-116a61982644/hdc1/" + logoPath);
-        GlobusOverlayAccessIO<DvObject> gsio = new GlobusOverlayAccessIO<DvObject>(datafile, null, "globus");
-        System.out.println("Size2 is " + gsio.retrieveSizeFromMedia());
-
-        System.out.println(
-                "NotValid: " + GlobusOverlayAccessIO.isValidIdentifier("globus", "globus://localid//../of/the/hill"));
-        System.out.println(
-                "ValidRemote: " + GlobusOverlayAccessIO.isValidIdentifier("globus", "globus://localid//of/the/hill"));
-        System.setProperty("dataverse.files.globus.managed", "true");
-        datafile.setStorageIdentifier("globus://" + baseStoreId + "//" + logoPath);
-        System.out.println("ValidLocal: "
-                + GlobusOverlayAccessIO.isValidIdentifier("globus", "globus://176e28068b0-1c3f80357c42"));
-
-        // We can read the storageIdentifier and get the driver
-        assertTrue(datafile.getStorageIdentifier()
-                .startsWith(DataAccess.getStorageDriverFromIdentifier(datafile.getStorageIdentifier())));
-        // We can get the driver type from it's ID
-        assertTrue(DataAccess.getDriverType("globus").equals(System.getProperty("dataverse.files.globus.type")));
-        // When we get a StorageIO for the file, it is the right type
-        StorageIO<DataFile> storageIO = DataAccess.getStorageIO(localDatafile);
-        assertTrue(storageIO instanceof GlobusOverlayAccessIO);
-        // When we use it, we can get properties like the remote store name
-        GlobusOverlayAccessIO<DataFile> globusIO = (GlobusOverlayAccessIO<DataFile>) storageIO;
-        assertTrue(
-                globusIO.getRemoteStoreName().equals(System.getProperty("dataverse.files.globus.remote-store-name")));
-
-        String location = globusIO.getStorageLocation();
-        assertEquals("globus:///" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + baseStoreId, location);
-/*
-        // TBD:
-        // And can get a temporary download URL for the main file
-        String signedURL = globusIO.generateTemporaryDownloadUrl(null, null, null);
-        System.out.println(signedURL);
-        // And the URL starts with the right stuff
-        assertTrue(signedURL.startsWith(System.getProperty("dataverse.files.globus." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH) + "/" + logoPath));
-        // And the signature is valid
-        // assertTrue(
-        // UrlSignerUtil.isValidUrl(signedURL, null, null,
-        // System.getProperty("dataverse.files.globus.secret-key")));
-        // And we get an unsigned URL with the right stuff with no key
-        System.clearProperty("dataverse.files.globus.secret-key");
-        String unsignedURL = globusIO.generateTemporaryDownloadUrl(null, null, null);
-        assertTrue(unsignedURL.equals(System.getProperty("dataverse.files.globus.base-url") + "/" + logoPath));
-*/
-        // Once we've opened, we can get the file size (only works if the call to Globus
-        // works)
-        globusIO.open(DataAccessOption.READ_ACCESS);
-        assertTrue(globusIO.getSize() > 0);
-        // If we ask for the path for an aux file, it is correct
-        System.out.println(Paths.get(System.getProperty("dataverse.files.file.directory", "/tmp/files"), authority,
-                identifier, baseStoreId + ".auxobject").toString());
-        System.out.println(globusIO.getAuxObjectAsPath("auxobject").toString());
-        assertTrue(Paths.get(System.getProperty("dataverse.files.file.directory", "/tmp/files"), authority, identifier,
-                baseStoreId + ".auxobject").equals(globusIO.getAuxObjectAsPath("auxobject")));
-        IOException thrown = assertThrows(IOException.class, () -> DataAccess.getStorageIO(localDatafile),
-                "Expected getStorageIO() to throw, but it didn't");
-        // 'test' is the driverId in the IOException messages
-        assertTrue(thrown.getMessage().contains("globus"));
-
-    }
-
-    @Test
-    void testRemoteOverlayIdentifierFormats() throws IOException {
-        System.clearProperty("dataverse.files.globus.managed");
-        datafile.setStorageIdentifier(
-                "globus://" + baseStoreId + "//d8c42580-6528-4605-9ad8-116a61982644/hdc1/" + logoPath);
-        assertTrue(DataAccess.isValidDirectStorageIdentifier(datafile.getStorageIdentifier()));
-        assertFalse(
-                DataAccess.isValidDirectStorageIdentifier(datafile.getStorageIdentifier().replace("globus", "bad")));
-        assertFalse(DataAccess.isValidDirectStorageIdentifier(localDatafile.getStorageIdentifier()));
-        System.setProperty("dataverse.files.globus.managed", "true");
-        assertTrue(DataAccess.isValidDirectStorageIdentifier(localDatafile.getStorageIdentifier()));
-
-    }
-
-}

From 1a96c566bccdf32aefeaca89898a3746b146fa08 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Sun, 3 Dec 2023 18:57:59 -0500
Subject: [PATCH 853/922] the kill switch for the real-time storageuse updates
 (just in case) and some related documentation (#8549)

---
 .../source/admin/collectionquotas.rst         |  2 +
 .../source/installation/config.rst            |  5 ++
 .../dataverse/ingest/IngestServiceBean.java   |  2 +-
 .../iq/dataverse/settings/JvmSettings.java    |  3 +
 .../storageuse/StorageUseServiceBean.java     | 58 ++++++-------------
 5 files changed, 30 insertions(+), 40 deletions(-)

diff --git a/doc/sphinx-guides/source/admin/collectionquotas.rst b/doc/sphinx-guides/source/admin/collectionquotas.rst
index 883b6cf0c93..2ce3132e2ba 100644
--- a/doc/sphinx-guides/source/admin/collectionquotas.rst
+++ b/doc/sphinx-guides/source/admin/collectionquotas.rst
@@ -1,3 +1,4 @@
+
 Storage Quotas for Collections
 ==============================
 
@@ -15,3 +16,4 @@ Please note that only the sizes of the main datafiles and the archival tab-delim
 
 When quotas are set and enforced, the users will be informed of the remaining storage allocation on the file upload page together with other upload and processing limits.
 
+Part of the new and experimental nature of this feature is that we don't know for the fact yet how well it will function in real life on a very busy production system, despite our best efforts to test it prior to the release. One specific issue is having to update the recorded storage use for every parent collection of the given dataset whenever new files are added. This includes updating the combined size of the root, top collection - which will need to be updated after *every* file upload. In an unlikely case that this will start causing problems with race conditions and database update conflicts, it is possible to disable these updates (and thus disable the storage quotas feature), by setting the :ref:`dataverse.storageuse.disable-storageuse-increments` JVM setting to true.
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 52ba35376ac..03eeff9dbb6 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -2489,6 +2489,11 @@ This setting was added to keep S3 direct upload lightweight. When that feature i
 
 See also :ref:`s3-direct-upload-features-disabled`.
 
+dataverse.storageuse.disable-storageuse-increments
+++++++++++++++++++++++++++++++++++++++++++++++++++
+
+This setting serves the role of an emergency "kill switch" that will disable maintaining the real time record of storage use for all the datasets and collections in the database. Because of the experimental nature of this feature (see :doc:`/admin/collectionquotas`) that hasn't been used in production setting as of this release, v6.1 this setting is provided in case these updates start causing database race conditions and conflicts on a busy server. 
+
 dataverse.auth.oidc.*
 +++++++++++++++++++++
 
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index 5efb4c06f48..233f746fb17 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -206,7 +206,7 @@ public List<DataFile> saveAndAddFilesToDataset(DatasetVersion version,
                 boolean unattached = false;
                 boolean savedSuccess = false;
                 if (dataFile.getOwner() == null) {
-                    // is it ever "unattached"? 
+                    // is it ever "attached"? 
                     // do we ever call this method with dataFile.getOwner() != null? 
                     // - we really shouldn't be, either. 
                     unattached = true;
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index cc3272413c7..7c65bba77d5 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -150,6 +150,9 @@ public enum JvmSettings {
     SCOPE_NETCDF(PREFIX, "netcdf"),
     GEO_EXTRACT_S3_DIRECT_UPLOAD(SCOPE_NETCDF, "geo-extract-s3-direct-upload"),
 
+    // STORAGE USE SETTINGS
+    SCOPE_STORAGEUSE(PREFIX, "storageuse"),
+    STORAGEUSE_DISABLE_UPDATES(SCOPE_STORAGEUSE, "disable-storageuse-increments"),
     ;
 
     private static final String SCOPE_SEPARATOR = ".";
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
index b542a7cd661..18e4ef49640 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
@@ -1,12 +1,14 @@
 package edu.harvard.iq.dataverse.storageuse;
 
 import edu.harvard.iq.dataverse.DvObjectContainer;
+import edu.harvard.iq.dataverse.settings.JvmSettings;
 import jakarta.ejb.Stateless;
 import jakarta.ejb.TransactionAttribute;
 import jakarta.ejb.TransactionAttributeType;
 import jakarta.inject.Named;
 import jakarta.persistence.EntityManager;
 import jakarta.persistence.PersistenceContext;
+import java.util.Optional;
 import java.util.logging.Logger;
 
 /**
@@ -37,31 +39,6 @@ public Long findStorageSizeByDvContainerId(Long dvObjectId) {
         return res == null ? 0L : res;
     }
     
-    public void incrementStorageSizeHierarchy(DvObjectContainer dvObject, Long filesize) {
-        incrementStorageSize(dvObject, filesize); 
-        DvObjectContainer parent = dvObject.getOwner();
-        while (parent != null) {
-            incrementStorageSize(parent, filesize);
-            parent = parent.getOwner();
-        }
-    }
-    
-    /**
-     * @param dvObject
-     * @param filesize 
-     */
-    public void incrementStorageSize(DvObjectContainer dvObject, Long filesize) {
-        StorageUse dvContainerSU = findByDvContainerId(dvObject.getId());
-        if (dvContainerSU != null) {
-            // @todo: named query
-            dvContainerSU.incrementSizeInBytes(filesize);
-            em.merge(dvContainerSU);
-        } else {
-            dvContainerSU = new StorageUse(dvObject, filesize); 
-            em.persist(dvContainerSU);
-        }
-    }
-    
     /**
      * Increments the recorded storage size for all the dvobject parents of a
      * datafile, recursively. 
@@ -71,20 +48,23 @@ public void incrementStorageSize(DvObjectContainer dvObject, Long filesize) {
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
     public void incrementStorageSizeRecursively(Long dvObjectContainerId, Long increment) {
         //@todo should throw exceptions if either parameter is null
-        String queryString = "WITH RECURSIVE uptree (id, owner_id) AS\n"
-                + "("
-                + "    SELECT id, owner_id\n"
-                + "    FROM dvobject\n"
-                + "    WHERE id=" + dvObjectContainerId + "\n"
-                + "    UNION ALL\n"
-                + "    SELECT dvobject.id, dvobject.owner_id\n"
-                + "    FROM dvobject\n"
-                + "    JOIN uptree ON dvobject.id = uptree.owner_id)\n"
-                + "UPDATE storageuse SET sizeinbytes=COALESCE(sizeinbytes,0)+" + increment + "\n"
-                + "FROM uptree\n"
-                + "WHERE dvobjectcontainer_id = uptree.id;";
-        
-        int parentsUpdated = em.createNativeQuery(queryString).executeUpdate();
+        Optional<Boolean> allow = JvmSettings.STORAGEUSE_DISABLE_UPDATES.lookupOptional(Boolean.class);
+        if (!(allow.isPresent() && allow.get())) {
+            String queryString = "WITH RECURSIVE uptree (id, owner_id) AS\n"
+                    + "("
+                    + "    SELECT id, owner_id\n"
+                    + "    FROM dvobject\n"
+                    + "    WHERE id=" + dvObjectContainerId + "\n"
+                    + "    UNION ALL\n"
+                    + "    SELECT dvobject.id, dvobject.owner_id\n"
+                    + "    FROM dvobject\n"
+                    + "    JOIN uptree ON dvobject.id = uptree.owner_id)\n"
+                    + "UPDATE storageuse SET sizeinbytes=COALESCE(sizeinbytes,0)+" + increment + "\n"
+                    + "FROM uptree\n"
+                    + "WHERE dvobjectcontainer_id = uptree.id;";
+
+            int parentsUpdated = em.createNativeQuery(queryString).executeUpdate();
+        }
         // @todo throw an exception if the number of parent dvobjects updated by
         // the query is < 2 - ? 
     }

From 0a536da0c42ed9654641985f1fd8dc20b461c16c Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 4 Dec 2023 09:46:59 -0500
Subject: [PATCH 854/922] a missing ref in the doc. #8549

---
 doc/sphinx-guides/source/installation/config.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 03eeff9dbb6..7cb321708a7 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -2489,6 +2489,8 @@ This setting was added to keep S3 direct upload lightweight. When that feature i
 
 See also :ref:`s3-direct-upload-features-disabled`.
 
+.. _dataverse.storageuse.disable-storageuse-increments:
+
 dataverse.storageuse.disable-storageuse-increments
 ++++++++++++++++++++++++++++++++++++++++++++++++++
 

From b20f198368615d7d8c4e798a25d6f68a6d0c4ed9 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Mon, 4 Dec 2023 11:27:27 -0500
Subject: [PATCH 855/922] Bump version to 6.1

---
 doc/sphinx-guides/source/conf.py      | 4 ++--
 doc/sphinx-guides/source/versions.rst | 3 ++-
 modules/dataverse-parent/pom.xml      | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py
index 0660ec3b071..64efc359e9a 100755
--- a/doc/sphinx-guides/source/conf.py
+++ b/doc/sphinx-guides/source/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '6.0'
+version = '6.1'
 # The full version, including alpha/beta/rc tags.
-release = '6.0'
+release = '6.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst
index 2000a2097f0..2cf7f46dc5e 100755
--- a/doc/sphinx-guides/source/versions.rst
+++ b/doc/sphinx-guides/source/versions.rst
@@ -7,7 +7,8 @@ Dataverse Software Documentation Versions
 This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases <https://github.com/IQSS/dataverse/releases>`__ page in our GitHub repo.
 
 - pre-release `HTML (not final!) <http://preview.guides.gdcc.io/en/develop/>`__ and `PDF (experimental!) <http://preview.guides.gdcc.io/_/downloads/en/develop/pdf/>`__ built from the :doc:`develop </developers/version-control>` branch :doc:`(how to contribute!) </developers/documentation>`
-- 6.0
+- 6.1
+- `6.0 </en/6.0/>`__
 - `5.14 </en/5.14/>`__
 - `5.13 </en/5.13/>`__
 - `5.12.1 </en/5.12.1/>`__
diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index db0fa46a952..7b305cad581 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -131,7 +131,7 @@
  
     <properties>
         <!-- This is a special Maven property name, do not change! -->
-        <revision>6.0</revision>
+        <revision>6.1</revision>
     
         <target.java.version>17</target.java.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

From 5f29144762c166c7856958497e24f629d53c92a0 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Mon, 4 Dec 2023 12:58:01 -0500
Subject: [PATCH 856/922] adding 6.1 release notes and removing .md files

---
 ...001-datasets-files-api-user-permissions.md |  13 --
 doc/release-notes/10060-api-changelog.md      |   3 -
 .../10093-signedUrl_improvements.md           |   5 -
 .../10104-dataset-citation-deaccessioned.md   |   1 -
 doc/release-notes/6.1-release-notes.md        | 195 ++++++++++++++++++
 .../9268-8349-oidc-improvements.md            |  43 ----
 doc/release-notes/9412-markdown-previewer.md  |   1 -
 doc/release-notes/9428-alternative-title.md   |   9 -
 doc/release-notes/9589-ds-configure-tool.md   |   1 -
 doc/release-notes/9590-intellij-redeploy.md   |   3 -
 .../9599-guestbook-at-request.md              |   2 -
 doc/release-notes/9635-solr-improvements.md   |   4 -
 doc/release-notes/9692-files-api-extension.md |   7 -
 .../9714-files-api-extension-filters.md       |  14 --
 .../9763-versions-api-improvements.md         |   8 -
 .../9785-files-api-extension-search-text.md   |   3 -
 .../9834-files-api-extension-counts.md        |   6 -
 ...oad-extension-new-file-access-endpoints.md |  14 --
 .../9852-files-api-extension-deaccession.md   |  12 --
 .../9880-info-api-zip-limit-embargo.md        |   5 -
 .../9907-files-api-counts-with-criteria.md    |  11 -
 doc/release-notes/9955-Signposting-updates.md |   7 -
 ...et-api-downloadsize-ignore-tabular-size.md |   9 -
 .../9972-files-api-filter-by-tabular-tags.md  |   3 -
 ...with-criteria-and-deaccessioned-support.md |  12 --
 25 files changed, 195 insertions(+), 196 deletions(-)
 delete mode 100644 doc/release-notes/10001-datasets-files-api-user-permissions.md
 delete mode 100644 doc/release-notes/10060-api-changelog.md
 delete mode 100644 doc/release-notes/10093-signedUrl_improvements.md
 delete mode 100644 doc/release-notes/10104-dataset-citation-deaccessioned.md
 create mode 100644 doc/release-notes/6.1-release-notes.md
 delete mode 100644 doc/release-notes/9268-8349-oidc-improvements.md
 delete mode 100644 doc/release-notes/9412-markdown-previewer.md
 delete mode 100644 doc/release-notes/9428-alternative-title.md
 delete mode 100644 doc/release-notes/9589-ds-configure-tool.md
 delete mode 100644 doc/release-notes/9590-intellij-redeploy.md
 delete mode 100644 doc/release-notes/9599-guestbook-at-request.md
 delete mode 100644 doc/release-notes/9635-solr-improvements.md
 delete mode 100644 doc/release-notes/9692-files-api-extension.md
 delete mode 100644 doc/release-notes/9714-files-api-extension-filters.md
 delete mode 100644 doc/release-notes/9763-versions-api-improvements.md
 delete mode 100644 doc/release-notes/9785-files-api-extension-search-text.md
 delete mode 100644 doc/release-notes/9834-files-api-extension-counts.md
 delete mode 100644 doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
 delete mode 100644 doc/release-notes/9852-files-api-extension-deaccession.md
 delete mode 100644 doc/release-notes/9880-info-api-zip-limit-embargo.md
 delete mode 100644 doc/release-notes/9907-files-api-counts-with-criteria.md
 delete mode 100644 doc/release-notes/9955-Signposting-updates.md
 delete mode 100644 doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
 delete mode 100644 doc/release-notes/9972-files-api-filter-by-tabular-tags.md
 delete mode 100644 doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md

diff --git a/doc/release-notes/10001-datasets-files-api-user-permissions.md b/doc/release-notes/10001-datasets-files-api-user-permissions.md
deleted file mode 100644
index 0aa75f9218a..00000000000
--- a/doc/release-notes/10001-datasets-files-api-user-permissions.md
+++ /dev/null
@@ -1,13 +0,0 @@
-- New query parameter `includeDeaccessioned` added to the getVersion endpoint (/api/datasets/{id}/versions/{versionId}) to consider deaccessioned versions when searching for versions.
-  
-
-- New endpoint to get user permissions on a dataset (/api/datasets/{id}/userPermissions). In particular, the user permissions that this API call checks, returned as booleans, are the following:
-
-  - Can view the unpublished dataset
-  - Can edit the dataset
-  - Can publish the dataset
-  - Can manage the dataset permissions
-  - Can delete the dataset draft
-
-
-- New permission check "canManageFilePermissions" added to the existing endpoint for getting user permissions on a file (/api/access/datafile/{id}/userPermissions).
\ No newline at end of file
diff --git a/doc/release-notes/10060-api-changelog.md b/doc/release-notes/10060-api-changelog.md
deleted file mode 100644
index 56ac96e3564..00000000000
--- a/doc/release-notes/10060-api-changelog.md
+++ /dev/null
@@ -1,3 +0,0 @@
-We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
-
-See also #10060.
diff --git a/doc/release-notes/10093-signedUrl_improvements.md b/doc/release-notes/10093-signedUrl_improvements.md
deleted file mode 100644
index 26a17c65e3f..00000000000
--- a/doc/release-notes/10093-signedUrl_improvements.md
+++ /dev/null
@@ -1,5 +0,0 @@
-A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
-
-SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
-
-Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
diff --git a/doc/release-notes/10104-dataset-citation-deaccessioned.md b/doc/release-notes/10104-dataset-citation-deaccessioned.md
deleted file mode 100644
index 0ba06d729c4..00000000000
--- a/doc/release-notes/10104-dataset-citation-deaccessioned.md
+++ /dev/null
@@ -1 +0,0 @@
-The getDatasetVersionCitation (/api/datasets/{id}/versions/{versionId}/citation) endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation.
diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
new file mode 100644
index 00000000000..c2b52ab34b8
--- /dev/null
+++ b/doc/release-notes/6.1-release-notes.md
@@ -0,0 +1,195 @@
+# Dataverse 6.1
+
+(If this note appears truncated on the GitHub Releases page, you can view it in full in the source tree: https://github.com/IQSS/dataverse/blob/master/doc/release-notes/6.1-release-notes.md)
+
+This release brings new features, enhancements, and bug fixes to the Dataverse software.
+Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
+
+## Release Highlights (Major Upgrades, Breaking Changes)
+
+This release contains major upgrades to core components. Detailed upgrade instructions can be found below.
+
+## Detailed Release Highlights, New Features and Use Case Scenarios
+
+### Dataverse installation can be now be configured to allow out-of-band upload
+- Installation can be now be configured to allow out-of-band upload by setting the `dataverse.files.<id>.upload-out-of-band` JVM option to `true`.
+By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+
+### Alternative Title is made repeatable.
+- One will need to update database with updated citation block.
+  `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
+- One will also need to update solr schema:
+ Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml`
+ Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
+
+Since Alternative Title is repeatable now, old json apis would not be compatable with a new version since value of alternative title has changed from simple string to an array.
+For example, instead "value": "Alternative Title", the value canbe "value": ["Alternative Title1", "Alternative Title2"]
+
+### Improvements in the /versions API
+- optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
+- a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
+- when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files.
+
+This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide.
+### The following API endpoints have been added:
+
+- /api/files/{id}/downloadCount
+- /api/files/{id}/dataTables 
+- /api/files/{id}/metadata/tabularTags New endpoint to set tabular file tags.
+- canManageFilePermissions (/access/datafile/{id}/userPermissions) Added for getting user permissions on a file.
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name).
+- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
+- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file.
+- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
+- deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
+- getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
+- getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.
+
+### Extended the existing endpoints:
+- getVersionFiles (/api/datasets/{id}/versions/{versionId}/files): Extended to support optional filtering by search text through the `searchText` query parameter. The search will be applied to the labels and descriptions of the dataset files. Added `tabularTagName` to return files to which the particular tabular tag has been added. Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files.
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain file counts. Added support for filtering by optional criteria query parameter:
+    - contentType
+    - accessStatus
+    - categoryName
+    - tabularTagName
+    - searchText
+- getDownloadSize ("api/datasets/{identifier}/versions/{versionId}/downloadsize"): Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files. Added a new optional query parameter "mode" 
+This parameter applies a filter criteria to the operation and supports the following values:
+    - All (Default): Includes both archival and original sizes for tabular files
+    - Archival: Includes only the archival size for tabular files
+    - Original: Includes only the original size for tabular files.
+- /api/datasets/{id}/versions/{versionId} New query parameter `includeDeaccessioned` added to consider deaccessioned versions when searching for versions.
+- /api/datasets/{id}/userPermissions Get user permissions on a dataset, in particular, the user permissions that this API call checks, returned as booleans, are the following:
+    - Can view the unpublished dataset
+    - Can edit the dataset
+    - Can publish the dataset
+    - Can manage the dataset permissions
+    - Can delete the dataset draft
+- getDatasetVersionCitation (/api/datasets/{id}/versions/{versionId}/citation) endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation.
+
+
+### DataFile API payload has been extended to include the following fields:
+- tabularData: Boolean field to know if the DataFile is of tabular type
+- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner)
+- friendlyType: String
+
+### The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination, ordering, and optional filtering
+- Access status: through the `accessStatus` query parameter, which supports the following values:
+    - Public
+    - Restricted
+    - EmbargoedThenRestricted
+    - EmbargoedThenPublic
+- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
+- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
+
+
+### Misc
+- Configure tools are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
+
+- Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
+The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions.
+
+- Dataverse's OAI_ORE Metadata Export format and archival BagIT exports
+(which include the OAI-ORE metadata export file) have been updated to include
+information about the dataset version state, e.g. RELEASED or DEACCESSIONED
+and to indicate which version of Dataverse was used to create the archival Bag.
+As part of the latter, the current OAI_ORE Metadata format has been given a 1.0.0
+version designation and it is expected that any future changes to the OAI_ORE export
+format will result in a version change and that tools such as DVUploader that can
+recreate datasets from archival Bags will start indicating which version(s) of the
+OAI_ORE format they can read.
+Dataverse installations that have been using archival Bags may wish to update any
+existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
+[archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
+to generate updated versions.
+
+- This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
+  - To fix #9952, we surround the license info with `<` and `>`.
+  - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
+  - To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
+
+- We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
+See also #10060.
+
+### Solr Improvements
+- As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
+
+Please see the "Installing Solr" section of the Installation Prerequisites guide.
+
+
+### Development
+- Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
+For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools
+
+- There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
+
+- A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
+  - SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
+  - Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
+
+## OpenID Connect Authentication Provider Improvements
+
+### Using MicroProfile Config For Provisioning
+
+With this release it is possible to provision a single OIDC-based authentication provider
+by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
+
+If you are using an external OIDC provider component as an identity management system and/or broker
+to other authentication providers such as Google, eduGain SAML and so on, this might make your
+life easier during instance setups and reconfiguration. You no longer need to generate the
+necessary JSON file.
+
+### Adding PKCE Support
+
+Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
+support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
+
+## Improved Testing
+
+With this release, we add a new type of testing to Dataverse: integration tests which are no end-to-end tests
+like our API tests. Starting with OIDC authentication support, we test regularly on CI for working condition
+of both OIDC login options in UI and API.
+
+The testing and development Keycloak realm has been updated with more users and compatibility with Keycloak 21.
+
+The support for setting JVM options during testing has been improved for developers. You now may add the
+`@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
+also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
+
+As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
+
+## New Configuration Options
+
+- dataverse.auth.oidc.enabled
+- dataverse.auth.oidc.client-id
+- dataverse.auth.oidc.client-secret
+- dataverse.auth.oidc.auth-server-url
+- dataverse.auth.oidc.pkce.enabled
+- dataverse.auth.oidc.pkce.method
+- dataverse.auth.oidc.title
+- dataverse.auth.oidc.subtitle
+- dataverse.auth.oidc.pkce.max-cache-size
+- dataverse.auth.oidc.pkce.max-cache-age
+
+## Installation
+
+If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it!
+
+Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club!
+
+You are also very welcome to join the [Global Dataverse Community Consortium](https://dataversecommunity.global) (GDCC).
+
+## Upgrade Instructions
+
+Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
+
+These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
+
+## Complete List of Changes
+
+For the complete list of code changes in this release, see the [6.1 Milestone](https://github.com/IQSS/dataverse/milestone/110?closed=1) in GitHub.
+
+## Getting Help
+
+For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
diff --git a/doc/release-notes/9268-8349-oidc-improvements.md b/doc/release-notes/9268-8349-oidc-improvements.md
deleted file mode 100644
index ddfc13e603c..00000000000
--- a/doc/release-notes/9268-8349-oidc-improvements.md
+++ /dev/null
@@ -1,43 +0,0 @@
-## OpenID Connect Authentication Provider Improvements
-
-### Using MicroProfile Config For Provisioning
-
-With this release it is possible to provision a single OIDC-based authentication provider
-by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
-
-If you are using an external OIDC provider component as an identity management system and/or broker
-to other authentication providers such as Google, eduGain SAML and so on, this might make your
-life easier during instance setups and reconfiguration. You no longer need to generate the
-necessary JSON file.
-
-### Adding PKCE Support
-
-Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
-support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
-
-## Improved Testing
-
-With this release, we add a new type of testing to Dataverse: integration tests which are no end-to-end tests
-like our API tests. Starting with OIDC authentication support, we test regularly on CI for working condition
-of both OIDC login options in UI and API.
-
-The testing and development Keycloak realm has been updated with more users and compatibility with Keycloak 21.
-
-The support for setting JVM options during testing has been improved for developers. You now may add the
-`@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
-also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
-
-As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
-
-## New Configuration Options
-
-- dataverse.auth.oidc.enabled
-- dataverse.auth.oidc.client-id
-- dataverse.auth.oidc.client-secret
-- dataverse.auth.oidc.auth-server-url
-- dataverse.auth.oidc.pkce.enabled
-- dataverse.auth.oidc.pkce.method
-- dataverse.auth.oidc.title
-- dataverse.auth.oidc.subtitle
-- dataverse.auth.oidc.pkce.max-cache-size
-- dataverse.auth.oidc.pkce.max-cache-age
diff --git a/doc/release-notes/9412-markdown-previewer.md b/doc/release-notes/9412-markdown-previewer.md
deleted file mode 100644
index 8faa2679fb0..00000000000
--- a/doc/release-notes/9412-markdown-previewer.md
+++ /dev/null
@@ -1 +0,0 @@
-There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
diff --git a/doc/release-notes/9428-alternative-title.md b/doc/release-notes/9428-alternative-title.md
deleted file mode 100644
index 3bc74f218b5..00000000000
--- a/doc/release-notes/9428-alternative-title.md
+++ /dev/null
@@ -1,9 +0,0 @@
-Alternative Title is made repeatable. 
-- One will need to update database with updated citation block.
-`curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
-- One will also need to update solr schema:
-Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml` 
-Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
-
-Since Alternative Title is repeatable now, old json apis would not be compatable with a new version since value of alternative title has changed from simple string to an array.
-For example, instead "value": "Alternative Title", the value canbe "value": ["Alternative Title1", "Alternative Title2"]  
diff --git a/doc/release-notes/9589-ds-configure-tool.md b/doc/release-notes/9589-ds-configure-tool.md
deleted file mode 100644
index 70ac5fcaa6a..00000000000
--- a/doc/release-notes/9589-ds-configure-tool.md
+++ /dev/null
@@ -1 +0,0 @@
-Configure tools are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
diff --git a/doc/release-notes/9590-intellij-redeploy.md b/doc/release-notes/9590-intellij-redeploy.md
deleted file mode 100644
index 07af352ece4..00000000000
--- a/doc/release-notes/9590-intellij-redeploy.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
-
-For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools
diff --git a/doc/release-notes/9599-guestbook-at-request.md b/doc/release-notes/9599-guestbook-at-request.md
deleted file mode 100644
index e9554b71fb4..00000000000
--- a/doc/release-notes/9599-guestbook-at-request.md
+++ /dev/null
@@ -1,2 +0,0 @@
-Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
-The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions. 
diff --git a/doc/release-notes/9635-solr-improvements.md b/doc/release-notes/9635-solr-improvements.md
deleted file mode 100644
index ad55ee3afe6..00000000000
--- a/doc/release-notes/9635-solr-improvements.md
+++ /dev/null
@@ -1,4 +0,0 @@
-- As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
-
-Please see the "Installing Solr" section of the Installation Prerequisites guide.
-
diff --git a/doc/release-notes/9692-files-api-extension.md b/doc/release-notes/9692-files-api-extension.md
deleted file mode 100644
index baa8e2f87cd..00000000000
--- a/doc/release-notes/9692-files-api-extension.md
+++ /dev/null
@@ -1,7 +0,0 @@
-The following API endpoints have been added:
-
-- /api/files/{id}/downloadCount
-- /api/files/{id}/dataTables
-- /access/datafile/{id}/userPermissions
-
-The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination and ordering
diff --git a/doc/release-notes/9714-files-api-extension-filters.md b/doc/release-notes/9714-files-api-extension-filters.md
deleted file mode 100644
index 034230efe61..00000000000
--- a/doc/release-notes/9714-files-api-extension-filters.md
+++ /dev/null
@@ -1,14 +0,0 @@
-The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by:
-
-- Access status: through the `accessStatus` query parameter, which supports the following values:
-
-  - Public
-  - Restricted
-  - EmbargoedThenRestricted
-  - EmbargoedThenPublic
-
-
-- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
-
-
-- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
diff --git a/doc/release-notes/9763-versions-api-improvements.md b/doc/release-notes/9763-versions-api-improvements.md
deleted file mode 100644
index 8d7f6c7a20a..00000000000
--- a/doc/release-notes/9763-versions-api-improvements.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Improvements in the /versions API
-
-- optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions;
-- a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output;
-- when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files. 
-
-This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide.
-
diff --git a/doc/release-notes/9785-files-api-extension-search-text.md b/doc/release-notes/9785-files-api-extension-search-text.md
deleted file mode 100644
index fb185e1c7af..00000000000
--- a/doc/release-notes/9785-files-api-extension-search-text.md
+++ /dev/null
@@ -1,3 +0,0 @@
-The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support optional filtering by search text through the `searchText` query parameter.
-
-The search will be applied to the labels and descriptions of the dataset files.
diff --git a/doc/release-notes/9834-files-api-extension-counts.md b/doc/release-notes/9834-files-api-extension-counts.md
deleted file mode 100644
index 3ec15d8bd36..00000000000
--- a/doc/release-notes/9834-files-api-extension-counts.md
+++ /dev/null
@@ -1,6 +0,0 @@
-Implemented the following new endpoints:
-
-- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name). 
-
-
-- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
diff --git a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
deleted file mode 100644
index f306ae2ab80..00000000000
--- a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md
+++ /dev/null
@@ -1,14 +0,0 @@
-Implemented the following new endpoints:
-
-- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file. 
-
-
-- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
-
-
-In addition, the DataFile API payload has been extended to include the following fields:
-
-- tabularData: Boolean field to know if the DataFile is of tabular type
-
-
-- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner) 
diff --git a/doc/release-notes/9852-files-api-extension-deaccession.md b/doc/release-notes/9852-files-api-extension-deaccession.md
deleted file mode 100644
index 55698580e3c..00000000000
--- a/doc/release-notes/9852-files-api-extension-deaccession.md
+++ /dev/null
@@ -1,12 +0,0 @@
-Extended the existing endpoints:
-
-- getVersionFiles (/api/datasets/{id}/versions/{versionId}/files)
-- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts)
-
-The above endpoints now accept a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files or file counts.
-
-Additionally, a new endpoint has been developed to support version deaccessioning through API (Given a dataset and a version).
-
-- deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession)
-
-Finally, the DataFile API payload has been extended to add the field "friendlyType"
diff --git a/doc/release-notes/9880-info-api-zip-limit-embargo.md b/doc/release-notes/9880-info-api-zip-limit-embargo.md
deleted file mode 100644
index d2afb139e72..00000000000
--- a/doc/release-notes/9880-info-api-zip-limit-embargo.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Implemented the following new endpoints:
-
-- getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
-
-- getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.
diff --git a/doc/release-notes/9907-files-api-counts-with-criteria.md b/doc/release-notes/9907-files-api-counts-with-criteria.md
deleted file mode 100644
index 07cd23daad0..00000000000
--- a/doc/release-notes/9907-files-api-counts-with-criteria.md
+++ /dev/null
@@ -1,11 +0,0 @@
-Extended the getVersionFileCounts endpoint (/api/datasets/{id}/versions/{versionId}/files/counts) to support filtering by criteria.
-
-In particular, the endpoint now accepts the following optional criteria query parameters:
-
-- contentType
-- accessStatus
-- categoryName
-- tabularTagName
-- searchText
-
-This filtering criteria is the same as the one for the getVersionFiles endpoint.
diff --git a/doc/release-notes/9955-Signposting-updates.md b/doc/release-notes/9955-Signposting-updates.md
deleted file mode 100644
index db0e27e51c5..00000000000
--- a/doc/release-notes/9955-Signposting-updates.md
+++ /dev/null
@@ -1,7 +0,0 @@
-This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
-
-To fix #9952, we surround the license info with `<` and `>`.
-
-To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
-
-To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
diff --git a/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md b/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
deleted file mode 100644
index 2ede679b361..00000000000
--- a/doc/release-notes/9958-dataset-api-downloadsize-ignore-tabular-size.md
+++ /dev/null
@@ -1,9 +0,0 @@
-Added a new optional query parameter "mode" to the "getDownloadSize" API endpoint ("api/datasets/{identifier}/versions/{versionId}/downloadsize").
-
-This parameter applies a filter criteria to the operation and supports the following values:
-
-- All (Default): Includes both archival and original sizes for tabular files
-
-- Archival: Includes only the archival size for tabular files 
-
-- Original: Includes only the original size for tabular files
diff --git a/doc/release-notes/9972-files-api-filter-by-tabular-tags.md b/doc/release-notes/9972-files-api-filter-by-tabular-tags.md
deleted file mode 100644
index 9c3fced1741..00000000000
--- a/doc/release-notes/9972-files-api-filter-by-tabular-tags.md
+++ /dev/null
@@ -1,3 +0,0 @@
-- New query parameter `tabularTagName` added to the getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) to return files to which the particular tabular tag has been added.
-
-- New endpoint to set tabular file tags via API: /api/files/{id}/metadata/tabularTags.
diff --git a/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md b/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
deleted file mode 100644
index 020224b2094..00000000000
--- a/doc/release-notes/9995-files-api-downloadsize-with-criteria-and-deaccessioned-support.md
+++ /dev/null
@@ -1,12 +0,0 @@
-Extended the getDownloadSize endpoint (/api/datasets/{id}/versions/{versionId}/downloadsize), including the following new features:
-
-- The endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned dataset versions when searching for versions to obtain the file total download size.
-
-
-- The endpoint now supports filtering by criteria. In particular, it accepts the following optional criteria query parameters:
-
-  - contentType
-  - accessStatus
-  - categoryName
-  - tabularTagName
-  - searchText

From b077d98a11e6957085757c54c48030ef33b50c30 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 4 Dec 2023 13:30:03 -0500
Subject: [PATCH 857/922] doc update, release note

---
 doc/release-notes/10162-globus-support.md          | 14 ++++++++++++++
 .../source/developers/big-data-support.rst         |  7 +++++--
 2 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 doc/release-notes/10162-globus-support.md

diff --git a/doc/release-notes/10162-globus-support.md b/doc/release-notes/10162-globus-support.md
new file mode 100644
index 00000000000..d64e72b70a1
--- /dev/null
+++ b/doc/release-notes/10162-globus-support.md
@@ -0,0 +1,14 @@
+Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible, and for referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/latest/developers/big-data-support.html#big-data-support)
+- Globus functionality remains 'experimental'/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incomatibilities.
+- The functionality is configured per store and replaces the previous single-S3-Connector-per-Dataverse-instance model
+- Adding files to a dataset, and accessing files is supported via the Dataverse user interface through a separate [dataverse-globus app](https://github.com/scholarsportal/dataverse-globus)
+- The functionality is also accessible via APIs (combining calls to the Dataverse and Globus APIs)
+
+Backward Incompatibilities:
+- The configuration for use of a Globus S3 Connector has changed and is aligned with the standard store configuration mechanism
+- The new functionality is incompatible with older versions of the globus-dataverse app and the Globus-related functionality in the UI will only function correctly if a Dataverse 6.1 compatible version of the dataverse-globus app is configured.
+
+New JVM Options:
+- A new 'globus' store type and associated store-related options have been added. These are described in the [File Storage Options section of the Dataverse Guides](https://guides.dataverse.org/en/latest/installation/config.html#file-storage-using-a-local-filesystem-and-or-swift-and-or-object-stores-and-or-trusted-remote-stores).
+
+Obsolete Settings: the :GlobusBasicToken, :GlobusEndpoint, and :GlobusStores settings are no longer used
diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
index d38f7f27a68..fe49f9f6150 100644
--- a/doc/sphinx-guides/source/developers/big-data-support.rst
+++ b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -152,8 +152,6 @@ Note: Globus file transfer is still experimental but feedback is welcome! See :r
 Users can transfer files via `Globus <ttps://www.globus.org>`_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s) 
 and a community-developed `dataverse-globus <https://github.com/scholarsportal/dataverse-globus>`_ app has been properly installed and configured.
 
-Due to differences in the access control models of a Dataverse installation and Globus, enabling the Globus capability on a store will disable the ability to restrict and embargo files in that store.
-
 Globus endpoints can be in a variety of places, from data centers to personal computers. 
 This means that from within the Dataverse software, a Globus transfer can feel like an upload or a download (with Globus Personal Connect running on your laptop, for example) or it can feel like a true transfer from one server to another (from a cluster in a data center into a Dataverse dataset or vice versa).
 
@@ -162,11 +160,16 @@ Globus transfer uses an efficient transfer mechanism and has additional features
 * robust file transfer capable of restarting after network or endpoint failures
 * third-party transfer, which enables a user accessing a Dataverse installation in their desktop browser to initiate transfer of their files from a remote endpoint (i.e. on a local high-performance computing cluster), directly to an S3 store managed by the Dataverse installation
 
+Note: Due to differences in the access control models of a Dataverse installation and Globus and the current Globus store model, Dataverse cannot enforce per-file-access restrictions.
+It is therefore recommended that a store be configured as public, which disables the ability to restrict and embargo files in that store, when Globus access is allowed.
+
 Dataverse supports three options for using Globus, two involving transfer to Dataverse-managed endpoints and one allowing Dataverse to reference files on remote endpoints.
 Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus
 S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint.
 Users will need a Globus account, which can be obtained via their institution or directly from Globus (at no cost).
 
+With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionlity related to ingest, previews, fixity hash validation, etc. are not available. (Using the S3-based endpoint, Dataverse has access via S3 and all functionlity normally associated with direct uploads to S3 is available.)
+
 For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. 
 Users will need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described in the dataset's Terms of Use and Access).
 

From 547d71c342e08ebdf674d8754dc072465ad20651 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 4 Dec 2023 14:31:07 -0500
Subject: [PATCH 858/922] #9464 add more detail to validation error message

---
 .../edu/harvard/iq/dataverse/DataverseServiceBean.java     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index ed46caf65a1..027e58d9263 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -1072,7 +1072,12 @@ public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
             schema.validate(new JSONObject(jsonInput)); // throws a ValidationException if this object is invalid
         } catch (ValidationException vx) {
             logger.info(BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " " + vx.getErrorMessage()); 
-            return BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " "  + vx.getErrorMessage();
+            String accumulatedexceptions = "";
+            for (ValidationException va : vx.getCausingExceptions()){
+                accumulatedexceptions = accumulatedexceptions + va;
+                accumulatedexceptions = accumulatedexceptions.replace("org.everit.json.schema.ValidationException:", " ");
+            }
+            return BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " "  + accumulatedexceptions;
         } catch (Exception ex) {            
             logger.info(BundleUtil.getStringFromBundle("dataverses.api.validate.json.exception") + ex.getLocalizedMessage());
             return BundleUtil.getStringFromBundle("dataverses.api.validate.json.exception") + ex.getLocalizedMessage();

From fc3ae08ec9335ac857af4d9c112e892255ef1c7a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 4 Dec 2023 14:44:00 -0500
Subject: [PATCH 859/922] adding documentation

---
 .../edu/harvard/iq/dataverse/DatasetPage.java |  21 ++
 .../harvard/iq/dataverse/api/Datasets.java    | 238 +++++++++++-------
 2 files changed, 163 insertions(+), 96 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 704c1d42228..f871d2e5198 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -6346,6 +6346,27 @@ public boolean isGlobusTransferRequested() {
         return globusTransferRequested;
     }
     
+    /**
+     * Analagous with the startDownload method, this method is called when the user
+     * tries to start a Globus transfer out (~download). The
+     * validateFilesForDownload call checks to see if there are some files that can
+     * be Globus transfered and, if so and there are no files that can't be
+     * transferre, this method will launch the globus transfer app. If there is a
+     * mix of files or if the guestbook popup is required, the method passes back to
+     * the UI so those popup(s) can be shown. Once they are, this method is called
+     * with the popupShown param true and the app will be shown.
+     * 
+     * @param transferAll - when called from the dataset Access menu, this should be
+     *                    true so that all files are included in the processing.
+     *                    When it is called from the file table, the current
+     *                    selection is used and the param should be false.
+     * @param popupShown  - This method is called twice if the the mixed files or
+     *                    guestbook popups are needed. On the first call, popupShown
+     *                    is false so that the transfer is not started and those
+     *                    popups can be shown. On the second call, popupShown is
+     *                    true and processing will occur as long as there are some
+     *                    valid files to transfer.
+     */
     public void startGlobusTransfer(boolean transferAll, boolean popupShown) {
         if (transferAll) {
             this.setSelectedFiles(workingVersion.getFileMetadatas());
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 5961b428bcb..ae576134be3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3444,90 +3444,34 @@ public Response getTimestamps(@Context ContainerRequestContext crc, @PathParam("
     }
 
 
-    @POST
-    @AuthRequired
-    @Path("{id}/addGlobusFiles")
-    @Consumes(MediaType.MULTIPART_FORM_DATA)
-    public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
-                                            @PathParam("id") String datasetId,
-                                            @FormDataParam("jsonData") String jsonData,
-                                            @Context UriInfo uriInfo
-    ) throws IOException, ExecutionException, InterruptedException {
-
-        logger.info(" ====  (api addGlobusFilesToDataset) jsonData   ====== " + jsonData);
-
-        if (!systemConfig.isHTTPUpload()) {
-            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled"));
-        }
-
-        // -------------------------------------
-        // (1) Get the user from the API key
-        // -------------------------------------
-        AuthenticatedUser authUser;
-        try {
-            authUser = getRequestAuthenticatedUserOrDie(crc);
-        } catch (WrappedResponse ex) {
-            return error(Response.Status.FORBIDDEN, BundleUtil.getStringFromBundle("file.addreplace.error.auth")
-            );
-        }
-
-        // -------------------------------------
-        // (2) Get the Dataset Id
-        // -------------------------------------
-        Dataset dataset;
-
-        try {
-            dataset = findDatasetOrDie(datasetId);
-        } catch (WrappedResponse wr) {
-            return wr.getResponse();
-        }
-        
-        JsonObject jsonObject = null;
-        try {
-            jsonObject = JsonUtil.getJsonObject(jsonData);
-        } catch (Exception ex) {
-            logger.fine("Error parsing json: " + jsonData + " " + ex.getMessage());
-            return badRequest("Error parsing json body");
-
-        }
-
-        //------------------------------------
-        // (2b) Make sure dataset does not have package file
-        // --------------------------------------
-
-        for (DatasetVersion dv : dataset.getVersions()) {
-            if (dv.isHasPackageFile()) {
-                return error(Response.Status.FORBIDDEN, BundleUtil.getStringFromBundle("file.api.alreadyHasPackageFile")
-                );
-            }
-        }
-
-
-        String lockInfoMessage = "Globus Upload API started ";
-        DatasetLock lock = datasetService.addDatasetLock(dataset.getId(), DatasetLock.Reason.GlobusUpload,
-                (authUser).getId(), lockInfoMessage);
-        if (lock != null) {
-            dataset.addLock(lock);
-        } else {
-            logger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId());
-        }
-
-
-        ApiToken token = authSvc.findApiTokenByUser(authUser);
-
-        if(uriInfo != null) {
-            logger.info(" ====  (api uriInfo.getRequestUri()) jsonData   ====== " + uriInfo.getRequestUri().toString());
-        }
-
-
-        String requestUrl = SystemConfig.getDataverseSiteUrlStatic();
-        
-        // Async Call
-        globusService.globusUpload(jsonObject, token, dataset, requestUrl, authUser);
-
-        return ok("Async call to Globus Upload started ");
-
-    }
+/****************************
+ * Globus Support Section:
+ * 
+ * Globus transfer in (upload) and out (download) involve three basic steps: The
+ * app is launched and makes a callback to the
+ * globusUploadParameters/globusDownloadParameters method to get all of the info
+ * needed to set up it's display.
+ * 
+ * At some point after that, the user will make a selection as to which files to
+ * transfer and the app will call requestGlobusUploadPaths/requestGlobusDownload
+ * to indicate a transfer is about to start. In addition to providing the
+ * details of where to transfer the files to/from, Dataverse also grants the
+ * Globus principal involved the relevant rw or r permission for the dataset.
+ * 
+ * Once the transfer is started, the app records the task id and sends it to
+ * Dataverse in the addGlobusFiles/monitorGlobusDownload call. Dataverse then
+ * monitors the transfer task and when it ultimately succeeds for fails it
+ * revokes the principal's permission and, for the transfer in case, adds the
+ * files to the dataset. (The dataset is locked until the transfer completes.)
+ * 
+ * (If no transfer is started within a specified timeout, permissions will
+ * automatically be revoked - see the GlobusServiceBean for details.)
+ *
+ * The option to reference a file at a remote endpoint (rather than transfer it)
+ * follows the first two steps of the process above but completes with a call to
+ * the normal /addFiles endpoint (as there is no transfer to monitor and the
+ * files can be added to the dataset immediately.)
+ */
 
     /**
      * Retrieve the parameters and signed URLs required to perform a globus
@@ -3630,11 +3574,11 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
     }
 
     /**
-     * Requests permissions for a given globus user to upload to the dataset
+     * Provides specific storageIdentifiers to use for each file amd requests permissions for a given globus user to upload to the dataset
      * 
      * @param crc
      * @param datasetId
-     * @param jsonData
+     * @param jsonData - an object that must include the id of the globus "principal" involved and the "numberOfFiles" that will be transferred.
      * @return
      * @throws IOException
      * @throws ExecutionException
@@ -3721,15 +3665,114 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
 
     }
 
-    /**
-     * Retrieve the parameters and signed URLs required to perform a globus
-     * transfer/download. This api endpoint is expected to be called as a signed
-     * callback after the globus-dataverse app/other app is launched, but it will
-     * accept other forms of authentication.
+    /** A method analogous to /addFiles that must also include the taskIdentifier of the transfer-in-progress to monitor
      * 
      * @param crc
      * @param datasetId
+     * @param jsonData - see /addFiles documentation, aditional "taskIdentifier" key in the main object is required.
+     * @param uriInfo
+     * @return
+     * @throws IOException
+     * @throws ExecutionException
+     * @throws InterruptedException
      */
+    @POST
+    @AuthRequired
+    @Path("{id}/addGlobusFiles")
+    @Consumes(MediaType.MULTIPART_FORM_DATA)
+    public Response addGlobusFilesToDataset(@Context ContainerRequestContext crc,
+                                            @PathParam("id") String datasetId,
+                                            @FormDataParam("jsonData") String jsonData,
+                                            @Context UriInfo uriInfo
+    ) throws IOException, ExecutionException, InterruptedException {
+
+        logger.info(" ====  (api addGlobusFilesToDataset) jsonData   ====== " + jsonData);
+
+        if (!systemConfig.isHTTPUpload()) {
+            return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled"));
+        }
+
+        // -------------------------------------
+        // (1) Get the user from the API key
+        // -------------------------------------
+        AuthenticatedUser authUser;
+        try {
+            authUser = getRequestAuthenticatedUserOrDie(crc);
+        } catch (WrappedResponse ex) {
+            return error(Response.Status.FORBIDDEN, BundleUtil.getStringFromBundle("file.addreplace.error.auth")
+            );
+        }
+
+        // -------------------------------------
+        // (2) Get the Dataset Id
+        // -------------------------------------
+        Dataset dataset;
+
+        try {
+            dataset = findDatasetOrDie(datasetId);
+        } catch (WrappedResponse wr) {
+            return wr.getResponse();
+        }
+        
+        JsonObject jsonObject = null;
+        try {
+            jsonObject = JsonUtil.getJsonObject(jsonData);
+        } catch (Exception ex) {
+            logger.fine("Error parsing json: " + jsonData + " " + ex.getMessage());
+            return badRequest("Error parsing json body");
+
+        }
+
+        //------------------------------------
+        // (2b) Make sure dataset does not have package file
+        // --------------------------------------
+
+        for (DatasetVersion dv : dataset.getVersions()) {
+            if (dv.isHasPackageFile()) {
+                return error(Response.Status.FORBIDDEN, BundleUtil.getStringFromBundle("file.api.alreadyHasPackageFile")
+                );
+            }
+        }
+
+
+        String lockInfoMessage = "Globus Upload API started ";
+        DatasetLock lock = datasetService.addDatasetLock(dataset.getId(), DatasetLock.Reason.GlobusUpload,
+                (authUser).getId(), lockInfoMessage);
+        if (lock != null) {
+            dataset.addLock(lock);
+        } else {
+            logger.log(Level.WARNING, "Failed to lock the dataset (dataset id={0})", dataset.getId());
+        }
+
+
+        ApiToken token = authSvc.findApiTokenByUser(authUser);
+
+        if(uriInfo != null) {
+            logger.info(" ====  (api uriInfo.getRequestUri()) jsonData   ====== " + uriInfo.getRequestUri().toString());
+        }
+
+
+        String requestUrl = SystemConfig.getDataverseSiteUrlStatic();
+        
+        // Async Call
+        globusService.globusUpload(jsonObject, token, dataset, requestUrl, authUser);
+
+        return ok("Async call to Globus Upload started ");
+
+    }
+    
+/**
+ * Retrieve the parameters and signed URLs required to perform a globus
+ * transfer/download. This api endpoint is expected to be called as a signed
+ * callback after the globus-dataverse app/other app is launched, but it will
+ * accept other forms of authentication.
+ * 
+ * @param crc
+ * @param datasetId
+ * @param locale
+ * @param downloadId - an id to a cached object listing the files involved. This is generated via Dataverse and provided to the dataverse-globus app in a signedURL.
+ * @return - JSON containing the parameters and URLs needed by the dataverse-globus app. The format is analogous to that for external tools. 
+ */
     @GET
     @AuthRequired
     @Path("{id}/globusDownloadParameters")
@@ -3815,12 +3858,14 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
 
     /**
      * Requests permissions for a given globus user to download the specified files
-     * the dataset
+     * the dataset and returns information about the paths to transfer from.
+     * 
+     * When called directly rather than in response to being given a downloadId, the jsonData can include a "fileIds" key with an array of file ids to transfer.
      * 
      * @param crc
      * @param datasetId
-     * @param jsonData
-     * @return
+     * @param jsonData - a JSON object that must include the id of the  Globus "principal" that will be transferring the files in the case where Dataverse manages the Globus endpoint. For remote endpoints, the principal is not required.
+     * @return - a JSON object containing a map of file ids to Globus endpoint/path
      * @throws IOException
      * @throws ExecutionException
      * @throws InterruptedException
@@ -3957,11 +4002,12 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
 
     /**
      * Monitors a globus download and removes permissions on the dir/dataset when
-     * done
+     * the specified transfer task is completed.
      * 
      * @param crc
      * @param datasetId
-     * @param jsonData
+     * @param jsonData  - a JSON Object containing the key "taskIdentifier" with the
+     *                  Globus task to monitor.
      * @return
      * @throws IOException
      * @throws ExecutionException

From 7697157ac98049dea45a2bd98193aad75e6037e1 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Mon, 4 Dec 2023 15:27:21 -0500
Subject: [PATCH 860/922] #9464 handle single errors

---
 .../edu/harvard/iq/dataverse/DataverseServiceBean.java     | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 027e58d9263..07e7fe615e2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -1077,7 +1077,12 @@ public String isDatasetJsonValid(String dataverseAlias, String jsonInput) {
                 accumulatedexceptions = accumulatedexceptions + va;
                 accumulatedexceptions = accumulatedexceptions.replace("org.everit.json.schema.ValidationException:", " ");
             }
-            return BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " "  + accumulatedexceptions;
+            if (!accumulatedexceptions.isEmpty()){
+                return BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " "  + accumulatedexceptions;
+            } else {
+                return BundleUtil.getStringFromBundle("dataverses.api.validate.json.failed") + " "  + vx.getErrorMessage();
+            }
+            
         } catch (Exception ex) {            
             logger.info(BundleUtil.getStringFromBundle("dataverses.api.validate.json.exception") + ex.getLocalizedMessage());
             return BundleUtil.getStringFromBundle("dataverses.api.validate.json.exception") + ex.getLocalizedMessage();

From 8ec61d084a81c7d5786bd583177b80255aa7e883 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 4 Dec 2023 15:58:21 -0500
Subject: [PATCH 861/922] cleanup, add method stubs, open for basestore,
 info->fine

---
 .../AbstractRemoteOverlayAccessIO.java        |  12 +-
 .../dataaccess/GlobusAccessibleStore.java     |   6 +
 .../dataaccess/GlobusOverlayAccessIO.java     | 142 ++++++++++++------
 3 files changed, 112 insertions(+), 48 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
index 16defc26a4f..8d058b7c9e3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
@@ -15,11 +15,8 @@
 
 import javax.net.ssl.SSLContext;
 
-import org.apache.http.Header;
 import org.apache.http.client.config.CookieSpecs;
 import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
@@ -30,15 +27,18 @@
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.protocol.HTTP;
 import org.apache.http.ssl.SSLContextBuilder;
-import org.apache.http.util.EntityUtils;
-
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
 
+
+/**
+ * A base class for StorageIO implementations supporting remote access. At present, that includes the RemoteOverlayAccessIO store and the newer GlobusOverlayAccessIO store. It primarily includes
+ * common methods for handling auxiliary files in the configured base store.
+ * @param <T>
+ */
 public abstract class AbstractRemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     protected static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index d827e40e807..e4d062f0619 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -6,7 +6,13 @@
 
 public interface GlobusAccessibleStore {
 
+    //Whether Dataverse manages access controls for the Globus endpoint or not.
     static final String MANAGED = "managed";
+    /*
+     * transfer and reference endpoint formats: <globusEndpointId/basePath>
+     * 
+     * REFERENCE_ENDPOINTS_WITH_BASEPATHS - reference endpoints separated by a comma
+     */
     static final String TRANSFER_ENDPOINT_WITH_BASEPATH = "transfer-endpoint-with-basepath";
     static final String GLOBUS_TOKEN = "globus-token";
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index e825af8cf30..7a6809cb2ff 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -2,12 +2,15 @@
 
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.Dataverse;
 import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.datavariable.DataVariable;
 import edu.harvard.iq.dataverse.globus.AccessToken;
 import edu.harvard.iq.dataverse.globus.GlobusServiceBean;
 import edu.harvard.iq.dataverse.util.UrlSignerUtil;
 import edu.harvard.iq.dataverse.util.json.JsonUtil;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
@@ -16,6 +19,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.List;
 import java.util.logging.Logger;
 
 import org.apache.http.client.ClientProtocolException;
@@ -32,9 +36,18 @@
 
 /**
  * @author qqmyers
- */
+ * 
+ * This class implements three related use cases, all of which leverage the underlying idea of using a base store (as with the Https RemoteOverlay store):
+ *   Managed - where Dataverse has control of the specified Globus endpoint and can set/remove permissions as needed to allow file transfers in/out:
+ *      File/generic endpoint - assumes Dataverse does not have access to the datafile contents
+ *      S3-Connector endpoint - assumes the datafiles are accessible via Globus and via S3 such that Dataverse can access to the datafile contents when needed.
+ *   Remote - where Dataverse references files that remain at remote Globus endpoints (as with the Https RemoteOverlay store) and cannot access to the datafile contents.
+ *   
+ *   Note that Globus endpoints can provide Http URLs to get file contents, so a future enhancement could potentially support datafile contents access in the Managed/File and Remote cases. 
+ *   
+ *    */
 /*
- * Globus Overlay Driver
+ * Globus Overlay Driver storageIdentifer format:
  * 
  * Remote: StorageIdentifier format:
  * <globusDriverId>://<baseStorageIdentifier>//<relativePath> 
@@ -47,11 +60,6 @@
  * 
  * Storage location:
  * <globusEndpointId/basepath>/<dataset authority>/<dataset identifier>/<baseStorageIdentifier>
- *
- * transfer and reference endpoint formats: 
- * <globusEndpointId/basePath>
- * 
- * reference endpoints separated by a comma
  * 
  */
 public class GlobusOverlayAccessIO<T extends DvObject> extends AbstractRemoteOverlayAccessIO<T> implements GlobusAccessibleStore {
@@ -115,7 +123,6 @@ private String retrieveGlobusAccessToken() {
         return accessToken.getOtherTokens().get(0).getAccessToken();
     }
 
-
     private void parsePath() {
         int filenameStart = path.lastIndexOf("/") + 1;
         String endpointWithBasePath = null;
@@ -126,9 +133,9 @@ private void parsePath() {
         }
         //String endpointWithBasePath = baseEndpointPath.substring(baseEndpointPath.lastIndexOf(DataAccess.SEPARATOR) + 3);
         int pathStart = endpointWithBasePath.indexOf("/");
-        logger.info("endpointWithBasePath: " + endpointWithBasePath);
+        logger.fine("endpointWithBasePath: " + endpointWithBasePath);
         endpointPath = "/" + (pathStart > 0 ? endpointWithBasePath.substring(pathStart + 1) : "");
-        logger.info("endpointPath: " + endpointPath);
+        logger.fine("endpointPath: " + endpointPath);
         
 
         if (isManaged() && (dvObject!=null)) {
@@ -146,7 +153,7 @@ private void parsePath() {
         if (filenameStart > 0) {
             relativeDirectoryPath = relativeDirectoryPath + path.substring(0, filenameStart);
         }
-        logger.info("relativeDirectoryPath finally: " + relativeDirectoryPath);
+        logger.fine("relativeDirectoryPath finally: " + relativeDirectoryPath);
         filename = path.substring(filenameStart);
         endpoint = pathStart > 0 ? endpointWithBasePath.substring(0, pathStart) : endpointWithBasePath;
 
@@ -171,7 +178,7 @@ protected void validatePath(String relPath) throws IOException {
         } else {
             try {
                 String endpoint = findMatchingEndpoint(relPath, allowedEndpoints);
-                logger.info(endpoint + "  " + relPath);
+                logger.fine(endpoint + "  " + relPath);
 
                 if (endpoint == null || !Paths.get(endpoint, relPath).normalize().startsWith(endpoint)) {
                     throw new IOException(
@@ -189,7 +196,6 @@ protected void validatePath(String relPath) throws IOException {
     public long retrieveSizeFromMedia() {
         parsePath();
         String globusAccessToken = retrieveGlobusAccessToken();
-        logger.info("GAT2: " + globusAccessToken);
         // Construct Globus URL
         URI absoluteURI = null;
         try {
@@ -198,13 +204,12 @@ public long retrieveSizeFromMedia() {
                     + "/ls?path=" + endpointPath + relativeDirectoryPath + "&filter=name:" + filename);
             HttpGet get = new HttpGet(absoluteURI);
 
-            logger.info("Token is " + globusAccessToken);
             get.addHeader("Authorization", "Bearer " + globusAccessToken);
             CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
             if (response.getStatusLine().getStatusCode() == 200) {
                 // Get reponse as string
                 String responseString = EntityUtils.toString(response.getEntity());
-                logger.info("Response from " + get.getURI().toString() + " is: " + responseString);
+                logger.fine("Response from " + get.getURI().toString() + " is: " + responseString);
                 JsonObject responseJson = JsonUtil.getJsonObject(responseString);
                 JsonArray dataArray = responseJson.getJsonArray("DATA");
                 if (dataArray != null && dataArray.size() != 0) {
@@ -214,7 +219,7 @@ public long retrieveSizeFromMedia() {
             } else {
                 logger.warning("Response from " + get.getURI().toString() + " was "
                         + response.getStatusLine().getStatusCode());
-                logger.info(EntityUtils.toString(response.getEntity()));
+                logger.fine(EntityUtils.toString(response.getEntity()));
             }
         } catch (URISyntaxException e) {
             // Should have been caught in validatePath
@@ -258,16 +263,15 @@ public void delete() throws IOException {
             absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/submission_id");
             HttpGet get = new HttpGet(absoluteURI);
 
-            logger.info("Token is " + globusAccessToken);
             get.addHeader("Authorization", "Bearer " + globusAccessToken);
             CloseableHttpResponse response = getSharedHttpClient().execute(get, localContext);
             if (response.getStatusLine().getStatusCode() == 200) {
                 // Get reponse as string
                 String responseString = EntityUtils.toString(response.getEntity());
-                logger.info("Response from " + get.getURI().toString() + " is: " + responseString);
+                logger.fine("Response from " + get.getURI().toString() + " is: " + responseString);
                 JsonObject responseJson = JsonUtil.getJsonObject(responseString);
                 String submissionId = responseJson.getString("value");
-                logger.info("submission_id for delete is: " + submissionId);
+                logger.fine("submission_id for delete is: " + submissionId);
                 absoluteURI = new URI("https://transfer.api.globusonline.org/v0.10/delete");
                 HttpPost post = new HttpPost(absoluteURI);
                 JsonObjectBuilder taskJsonBuilder = Json.createObjectBuilder();
@@ -277,30 +281,30 @@ public void delete() throws IOException {
                 post.setHeader("Content-Type", "application/json");
                 post.addHeader("Authorization", "Bearer " + globusAccessToken);
                 String taskJson= JsonUtil.prettyPrint(taskJsonBuilder.build());
-                logger.info("Sending: " + taskJson);
+                logger.fine("Sending: " + taskJson);
                 post.setEntity(new StringEntity(taskJson, "utf-8"));
                 CloseableHttpResponse postResponse = getSharedHttpClient().execute(post, localContext);
                 int statusCode=postResponse.getStatusLine().getStatusCode();
-                logger.info("Response :" + statusCode + ": " +postResponse.getStatusLine().getReasonPhrase());
+                logger.fine("Response :" + statusCode + ": " +postResponse.getStatusLine().getReasonPhrase());
                 switch (statusCode) {
                 case 202:
                     // ~Success - delete task was accepted
-                    logger.info("Globus delete initiated: " + EntityUtils.toString(postResponse.getEntity()));
+                    logger.fine("Globus delete initiated: " + EntityUtils.toString(postResponse.getEntity()));
                     break;
                 case 200:
                     // Duplicate - delete task was already accepted
-                    logger.info("Duplicate Globus delete: " + EntityUtils.toString(postResponse.getEntity()));
+                    logger.warning("Duplicate Globus delete: " + EntityUtils.toString(postResponse.getEntity()));
                     break;
                 default:
                     logger.warning("Response from " + post.getURI().toString() + " was "
                             + postResponse.getStatusLine().getStatusCode());
-                    logger.info(EntityUtils.toString(postResponse.getEntity()));
+                    logger.fine(EntityUtils.toString(postResponse.getEntity()));
                 }
 
             } else {
                 logger.warning("Response from " + get.getURI().toString() + " was "
                         + response.getStatusLine().getStatusCode());
-                logger.info(EntityUtils.toString(response.getEntity()));
+                logger.fine(EntityUtils.toString(response.getEntity()));
             }
         } catch (Exception e) {
             logger.warning(e.getMessage());
@@ -383,7 +387,7 @@ public String getStorageLocation() throws IOException {
      */
     protected void configureGlobusEndpoints() throws IOException {
         allowedEndpoints = getAllowedEndpoints(this.driverId);
-        logger.info("Set allowed endpoints: " + Arrays.toString(allowedEndpoints));
+        logger.fine("Set allowed endpoints: " + Arrays.toString(allowedEndpoints));
     }
     
     private static String[] getAllowedEndpoints(String driverId) throws IOException {
@@ -409,37 +413,91 @@ private static String[] getAllowedEndpoints(String driverId) throws IOException
 
 
     @Override
-    public void open(DataAccessOption... option) throws IOException {
-        // TODO Auto-generated method stub
-        
-    }
+    public void open(DataAccessOption... options) throws IOException {
+
+        baseStore.open(options);
+
+        DataAccessRequest req = this.getRequest();
+
+        if (isWriteAccessRequested(options)) {
+            isWriteAccess = true;
+            isReadAccess = false;
+        } else {
+            isWriteAccess = false;
+            isReadAccess = true;
+        }
+
+        if (dvObject instanceof DataFile) {
+            String storageIdentifier = dvObject.getStorageIdentifier();
+
+            DataFile dataFile = this.getDataFile();
+
+            if (req != null && req.getParameter("noVarHeader") != null) {
+                baseStore.setNoVarHeader(true);
+            }
+
+            if (storageIdentifier == null || "".equals(storageIdentifier)) {
+                throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile.");
+            }
+
+            logger.fine("StorageIdentifier is: " + storageIdentifier);
+
+            if (isReadAccess) {
+                if (dataFile.getFilesize() >= 0) {
+                    this.setSize(dataFile.getFilesize());
+                } else {
+                    logger.fine("Setting size");
+                    this.setSize(retrieveSizeFromMedia());
+                }
+                // Only applies for the S3 Connector case (where we could have run an ingest)
+                if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values")
+                        && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
+
+                    List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
+                    String varHeaderLine = generateVariableHeader(datavariables);
+                    this.setVarHeader(varHeaderLine);
+                }
+
+            }
 
+            this.setMimeType(dataFile.getContentType());
+
+            try {
+                this.setFileName(dataFile.getFileMetadata().getLabel());
+            } catch (Exception ex) {
+                this.setFileName("unknown");
+            }
+        } else if (dvObject instanceof Dataset) {
+            throw new IOException(
+                    "Data Access: " + this.getClass().getName() + " does not support dvObject type Dataverse yet");
+        } else if (dvObject instanceof Dataverse) {
+            throw new IOException(
+                    "Data Access: " + this.getClass().getName() + " does not support dvObject type Dataverse yet");
+        }
+    }
 
     @Override
     public Path getFileSystemPath() throws IOException {
-        // TODO Auto-generated method stub
-        return null;
+        throw new UnsupportedDataAccessOperationException(
+                this.getClass().getName() + ": savePath() not implemented in this storage driver.");
     }
 
-
     @Override
     public void savePath(Path fileSystemPath) throws IOException {
-        // TODO Auto-generated method stub
-        
+        throw new UnsupportedDataAccessOperationException(
+                this.getClass().getName() + ": savePath() not implemented in this storage driver.");
     }
 
-
     @Override
     public void saveInputStream(InputStream inputStream) throws IOException {
-        // TODO Auto-generated method stub
-        
+        throw new UnsupportedDataAccessOperationException(
+                this.getClass().getName() + ": savePath() not implemented in this storage driver.");
     }
 
-
     @Override
     public void saveInputStream(InputStream inputStream, Long filesize) throws IOException {
-        // TODO Auto-generated method stub
-        
+        throw new UnsupportedDataAccessOperationException(
+                this.getClass().getName() + ": savePath() not implemented in this storage driver.");
     }
-    
+
 }

From 38c120e13d2e1276324b903be58306520168b577 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 4 Dec 2023 18:21:41 -0500
Subject: [PATCH 862/922] cleanup, delete obsolete methods, change to private,
 info->fine

---
 .../harvard/iq/dataverse/api/Datasets.java    |   4 +-
 .../dataverse/globus/GlobusServiceBean.java   | 461 +++---------------
 .../iq/dataverse/settings/JvmSettings.java    |   2 +-
 src/main/webapp/globus.xhtml                  |  30 --
 4 files changed, 78 insertions(+), 419 deletions(-)
 delete mode 100644 src/main/webapp/globus.xhtml

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index ae576134be3..cb57acd3b86 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -3543,7 +3543,7 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
         } else {
             params.add("referenceEndpointsWithPaths", referenceEndpointsWithPaths);
         }
-        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class);
+        int timeoutSeconds = JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class);
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         String requestCallName = managed ? "requestGlobusTransferPaths" : "requestGlobusReferencePaths";
         allowedApiCalls.add(
@@ -3833,7 +3833,7 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P
             params.add("endpoint", transferEndpoint);
         }
         params.add("files", files);
-        int timeoutSeconds = JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class);
+        int timeoutSeconds = JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class);
         JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
         allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "monitorGlobusDownload")
                 .add(URLTokenUtil.HTTP_METHOD, "POST")
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 0c991424ce9..37959188857 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -97,34 +97,6 @@ public class GlobusServiceBean implements java.io.Serializable {
     private static final Logger logger = Logger.getLogger(GlobusServiceBean.class.getCanonicalName());
     private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss");
 
-    private String code;
-    private String userTransferToken;
-    private String state;
-
-    public String getState() {
-        return state;
-    }
-
-    public void setState(String state) {
-        this.state = state;
-    }
-
-    public String getCode() {
-        return code;
-    }
-
-    public void setCode(String code) {
-        this.code = code;
-    }
-
-    public String getUserTransferToken() {
-        return userTransferToken;
-    }
-
-    public void setUserTransferToken(String userTransferToken) {
-        this.userTransferToken = userTransferToken;
-    }
-
     private String getRuleId(GlobusEndpoint endpoint, String principal, String permissions)
             throws MalformedURLException {
 
@@ -152,33 +124,6 @@ private String getRuleId(GlobusEndpoint endpoint, String principal, String permi
         return null;
     }
 
-    /*
-     * public void updatePermision(AccessToken clientTokenUser, String directory,
-     * String principalType, String perm) throws MalformedURLException { if
-     * (directory != null && !directory.equals("")) { directory = directory + "/"; }
-     * logger.info("Start updating permissions." + " Directory is " + directory);
-     * String globusEndpoint =
-     * settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusEndpoint, "");
-     * ArrayList<String> rules = checkPermisions(clientTokenUser, directory,
-     * globusEndpoint, principalType, null); logger.info("Size of rules " +
-     * rules.size()); int count = 0; while (count < rules.size()) {
-     * logger.info("Start removing rules " + rules.get(count)); Permissions
-     * permissions = new Permissions(); permissions.setDATA_TYPE("access");
-     * permissions.setPermissions(perm); permissions.setPath(directory);
-     * 
-     * Gson gson = new GsonBuilder().create(); URL url = new
-     * URL("https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint
-     * + "/access/" + rules.get(count));
-     * logger.info("https://transfer.api.globusonline.org/v0.10/endpoint/" +
-     * globusEndpoint + "/access/" + rules.get(count)); MakeRequestResponse result =
-     * makeRequest(url, "Bearer",
-     * clientTokenUser.getOtherTokens().get(0).getAccessToken(), "PUT",
-     * gson.toJson(permissions)); if (result.status != 200) {
-     * logger.warning("Cannot update access rule " + rules.get(count)); } else {
-     * logger.info("Access rule " + rules.get(count) + " was updated"); } count++; }
-     * }
-     */
-
     /**
      * Call to delete a globus rule related to the specified dataset.
      * 
@@ -214,6 +159,13 @@ public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger
         }
     }
 
+    /** Request read/write access for the specified principal and generate a list of accessible paths for new files for the specified dataset.
+     * 
+     * @param principal - the id of the Globus principal doing the transfer
+     * @param dataset
+     * @param numberOfPaths - how many files are to be transferred
+     * @return
+     */
     public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int numberOfPaths) {
 
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
@@ -278,6 +230,12 @@ private int requestPermission(GlobusEndpoint endpoint, Dataset dataset, Permissi
         }
     }
 
+    /** Given an array of remote files to be referenced in the dataset, create a set of valid storage identifiers and return a map of the remote file paths to storage identifiers.
+     * 
+     * @param dataset
+     * @param referencedFiles - a JSON array of remote files to be referenced in the dataset - each should be a string with the <Globus endpoint>/path/to/file
+     * @return - a map of supplied paths to valid storage identifiers
+     */
     public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray referencedFiles) {
         String driverId = dataset.getEffectiveStorageDriverId();
         JsonArray endpoints = GlobusAccessibleStore.getReferenceEndpointsWithPaths(driverId);
@@ -304,39 +262,38 @@ public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray ref
         return fileMap.build();
     }
 
+    
+    /** A cache of temporary permission requests - for upload (rw) and download (r) access.
+     * When a temporary permission request is created, it is added to the cache. After GLOBUS_CACHE_MAXAGE minutes, if a transfer has not been started, the permission will be revoked/deleted.
+     * (If a transfer has been started, the permission will not be revoked/deleted until the transfer is complete. This is handled in other methods.)
+     */
     // Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
             .expireAfterWrite(
-                    Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
+                    Duration.of(JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
             .scheduler(Scheduler.systemScheduler()).evictionListener((ruleId, datasetId, cause) -> {
                 // Delete rules that expire
-                logger.info("Rule " + ruleId + " expired");
+                logger.fine("Rule " + ruleId + " expired");
                 Dataset dataset = datasetSvc.find(datasetId);
                 deletePermission((String) ruleId, dataset, logger);
             })
 
             .build();
 
+    //Convenience method to add a temporary permission request to the cache - allows logging of temporary permission requests
     private void monitorTemporaryPermissions(String ruleId, long datasetId) {
-        logger.info("Adding rule " + ruleId + " for dataset " + datasetId);
+        logger.fine("Adding rule " + ruleId + " for dataset " + datasetId);
         rulesCache.put(ruleId, datasetId);
     }
 
-    public boolean getSuccessfulTransfers(AccessToken clientTokenUser, String taskId) throws MalformedURLException {
-
-        URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId
-                + "/successful_transfers");
-
-        MakeRequestResponse result = makeRequest(url, "Bearer",
-                clientTokenUser.getOtherTokens().get(0).getAccessToken(), "GET", null);
-
-        if (result.status == 200) {
-            logger.info(" SUCCESS ====== ");
-            return true;
-        }
-        return false;
-    }
-
+/** Call the Globus API to get info about the transfer.
+ * 
+ * @param accessToken
+ * @param taskId - the Globus task id supplied by the user
+ * @param globusLogger - the transaction-specific logger to use (separate log files are created in general, some calls may use the class logger)
+ * @return
+ * @throws MalformedURLException
+ */
     public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger) throws MalformedURLException {
 
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId);
@@ -356,6 +313,11 @@ public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger
         return task;
     }
 
+    /** Globus call to get an access token for the user using the long-term token we hold.
+     * 
+     * @param globusBasicToken - the base64 encoded Globus Basic token comprised of the <Globus user id>:<key>
+     * @return - a valid Globus access token
+     */
     public static AccessToken getClientToken(String globusBasicToken) {
         URL url;
         AccessToken clientTokenUser = null;
@@ -375,36 +337,7 @@ public static AccessToken getClientToken(String globusBasicToken) {
         return clientTokenUser;
     }
 
-    public AccessToken getAccessToken(HttpServletRequest origRequest, String globusBasicToken)
-            throws UnsupportedEncodingException, MalformedURLException {
-        String serverName = origRequest.getServerName();
-        if (serverName.equals("localhost")) {
-            logger.severe("Changing localhost to utoronto");
-            serverName = "utl-192-123.library.utoronto.ca";
-        }
-
-        String redirectURL = "https://" + serverName + "/globus.xhtml";
-
-        redirectURL = URLEncoder.encode(redirectURL, "UTF-8");
-
-        URL url = new URL("https://auth.globus.org/v2/oauth2/token?code=" + code + "&redirect_uri=" + redirectURL
-                + "&grant_type=authorization_code");
-        logger.info(url.toString());
-
-        MakeRequestResponse result = makeRequest(url, "Basic", globusBasicToken, "POST", null);
-        AccessToken accessTokenUser = null;
-
-        if (result.status == 200) {
-            logger.info("Access Token: \n" + result.toString());
-            accessTokenUser = parseJson(result.jsonResponse, AccessToken.class, true);
-            logger.info(accessTokenUser.getAccessToken());
-        }
-
-        return accessTokenUser;
-
-    }
-
-    public static MakeRequestResponse makeRequest(URL url, String authType, String authCode, String method,
+    private static MakeRequestResponse makeRequest(URL url, String authType, String authCode, String method,
             String jsonString) {
         String str = null;
         HttpURLConnection connection = null;
@@ -412,9 +345,8 @@ public static MakeRequestResponse makeRequest(URL url, String authType, String a
         try {
             connection = (HttpURLConnection) url.openConnection();
             // Basic
-            // NThjMGYxNDQtN2QzMy00ZTYzLTk3MmUtMjljNjY5YzJjNGJiOktzSUVDMDZtTUxlRHNKTDBsTmRibXBIbjZvaWpQNGkwWVVuRmQyVDZRSnc9
             logger.info(authType + " " + authCode);
-            logger.info("For URL: " + url.toString());
+            logger.fine("For URL: " + url.toString());
             connection.setRequestProperty("Authorization", authType + " " + authCode);
             // connection.setRequestProperty("Content-Type",
             // "application/x-www-form-urlencoded");
@@ -422,7 +354,7 @@ public static MakeRequestResponse makeRequest(URL url, String authType, String a
             if (jsonString != null) {
                 connection.setRequestProperty("Content-Type", "application/json");
                 connection.setRequestProperty("Accept", "application/json");
-                logger.info(jsonString);
+                logger.fine(jsonString);
                 connection.setDoOutput(true);
 
                 OutputStreamWriter wr = new OutputStreamWriter(connection.getOutputStream());
@@ -431,24 +363,21 @@ public static MakeRequestResponse makeRequest(URL url, String authType, String a
             }
 
             status = connection.getResponseCode();
-            logger.info("Status now " + status);
+            logger.fine("Status now " + status);
             InputStream result = connection.getInputStream();
             if (result != null) {
-                logger.info("Result is not null");
                 str = readResultJson(result).toString();
-                logger.info("str is ");
-                logger.info(result.toString());
+                logger.fine("str is " + result.toString());
             } else {
-                logger.info("Result is null");
+                logger.fine("Result is null");
                 str = null;
             }
 
-            logger.info("status: " + status);
+            logger.fine("status: " + status);
         } catch (IOException ex) {
-            logger.info("IO");
             logger.severe(ex.getMessage());
-            logger.info(ex.getCause().toString());
-            logger.info(ex.getStackTrace().toString());
+            logger.fine(ex.getCause().toString());
+            logger.fine(ex.getStackTrace().toString());
         } finally {
             if (connection != null) {
                 connection.disconnect();
@@ -461,16 +390,14 @@ public static MakeRequestResponse makeRequest(URL url, String authType, String a
 
     private static StringBuilder readResultJson(InputStream in) {
         StringBuilder sb = null;
-        try {
-
-            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) {
             sb = new StringBuilder();
             String line;
             while ((line = br.readLine()) != null) {
                 sb.append(line + "\n");
             }
             br.close();
-            logger.info(sb.toString());
+            logger.fine(sb.toString());
         } catch (IOException e) {
             sb = null;
             logger.severe(e.getMessage());
@@ -495,31 +422,6 @@ private static <T> T parseJson(String sb, Class<T> jsonParserClass, boolean nami
         }
     }
 
-    public String getDirectory(String datasetId) {
-        Dataset dataset = null;
-        String directory = null;
-        try {
-            dataset = datasetSvc.find(Long.parseLong(datasetId));
-            if (dataset == null) {
-                logger.severe("Dataset not found " + datasetId);
-                return null;
-            }
-            String storeId = dataset.getStorageIdentifier();
-            storeId.substring(storeId.indexOf("//") + 1);
-            directory = storeId.substring(storeId.indexOf("//") + 1);
-            logger.info(storeId);
-            logger.info(directory);
-            logger.info("Storage identifier:" + dataset.getIdentifierForFileStorage());
-            return directory;
-
-        } catch (NumberFormatException nfe) {
-            logger.severe(nfe.getMessage());
-
-            return null;
-        }
-
-    }
-
     static class MakeRequestResponse {
         public String jsonResponse;
         public int status;
@@ -531,53 +433,26 @@ static class MakeRequestResponse {
 
     }
 
-    /*
-     * unused - may be needed for S3 case private MakeRequestResponse
-     * findDirectory(String directory, String clientToken, String globusEndpoint)
-     * throws MalformedURLException { URL url = new
-     * URL(" https://transfer.api.globusonline.org/v0.10/endpoint/" + globusEndpoint
-     * + "/ls?path=" + directory + "/");
-     * 
-     * MakeRequestResponse result = makeRequest(url, "Bearer", clientToken, "GET",
-     * null); logger.info("find directory status:" + result.status);
-     * 
-     * return result; }
-     */
 
-    /*
-     * public boolean giveGlobusPublicPermissions(Dataset dataset) throws
-     * UnsupportedEncodingException, MalformedURLException {
-     * 
-     * GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
-     * 
-     * 
-     * MakeRequestResponse status = findDirectory(endpoint.getBasePath(),
-     * endpoint.getClientToken(), endpoint.getId());
-     * 
-     * if (status.status == 200) {
-     * 
-     * int perStatus = givePermission("all_authenticated_users", "", "r", dataset);
-     * logger.info("givePermission status " + perStatus); if (perStatus == 409) {
-     * logger.info("Permissions already exist or limit was reached"); } else if
-     * (perStatus == 400) { logger.info("No directory in Globus"); } else if
-     * (perStatus != 201 && perStatus != 200) {
-     * logger.info("Cannot give read permission"); return false; }
+    /**
+     * Cache of open download Requests This cache keeps track of the set of files
+     * selected for transfer out (download) via Globus. It is a means of
+     * transferring the list from the DatasetPage, where it is generated via user UI
+     * actions, and the Datasets/globusDownloadParameters API.
      * 
-     * } else if (status.status == 404) {
-     * logger.info("There is no globus directory"); } else {
-     * logger.severe("Cannot find directory in globus, status " + status); return
-     * false; }
+     * Nominally, the dataverse-globus app will call that API endpoint and then
+     * /requestGlobusDownload, at which point the cached info is sent to the app. If
+     * the app doesn't call within 5 minutes (the time allowed to call
+     * /globusDownloadParameters) + GLOBUS_CACHE_MAXAGE minutes (a ~longer period
+     * giving the user time to make choices in the app), the cached info is deleted.
      * 
-     * return true; }
      */
-
-    // Single cache of open rules/permission requests
     private final Cache<String, JsonObject> downloadCache = Caffeine.newBuilder()
             .expireAfterWrite(
-                    Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class) + 5, ChronoUnit.MINUTES))
+                    Duration.of(JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class) + 5, ChronoUnit.MINUTES))
             .scheduler(Scheduler.systemScheduler()).evictionListener((downloadId, datasetId, cause) -> {
                 // Delete downloads that expire
-                logger.info("Download for " + downloadId + " expired");
+                logger.fine("Download for " + downloadId + " expired");
             })
 
             .build();
@@ -600,11 +475,18 @@ public int setPermissionForDownload(Dataset dataset, String principal) {
         return requestPermission(endpoint, dataset, permissions);
     }
 
-    // Generates the URL to launch the Globus app
+    // Generates the URL to launch the Globus app for upload
     public String getGlobusAppUrlForDataset(Dataset d) {
         return getGlobusAppUrlForDataset(d, true, null);
     }
 
+    /** Generated the App URl for upload (in) or download (out)
+     * 
+     * @param d - the dataset involved
+     * @param upload - boolean, true for upload, false for download
+     * @param dataFiles - a list of the DataFiles to be downloaded
+     * @return
+     */
     public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<DataFile> dataFiles) {
         String localeCode = session.getLocaleCode();
         ApiToken apiToken = null;
@@ -654,7 +536,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<DataFile
         appUrl = appUrl + "&callback=" + Base64.getEncoder().encodeToString(StringUtils.getBytesUtf8(callback));
 
         String finalUrl = tokenUtil.replaceTokensWithValues(appUrl);
-        logger.info("Calling app: " + finalUrl);
+        logger.fine("Calling app: " + finalUrl);
         return finalUrl;
     }
 
@@ -676,7 +558,7 @@ public JsonObject getFilesMap(List<DataFile> dataFiles, Dataset d) {
         return filesBuilder.build();
     }
 
-    public String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List<DataFile> downloadDFList) {
+    private String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List<DataFile> downloadDFList) {
         return URLTokenUtil.getScriptForUrl(getGlobusAppUrlForDataset(dataset, false, downloadDFList));
 
     }
@@ -718,7 +600,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
         GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
         GlobusTask task = getTask(endpoint.getClientToken(), taskIdentifier, globusLogger);
         String ruleId = getRuleId(endpoint, task.getOwner_id(), "rw");
-        logger.info("Found rule: " + ruleId);
+        logger.fine("Found rule: " + ruleId);
         if (ruleId != null) {
             Long datasetId = rulesCache.getIfPresent(ruleId);
             if (datasetId != null) {
@@ -812,8 +694,8 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                     // calculateMissingMetadataFields: checksum, mimetype
                     JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, globusLogger);
                     JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files");
-                    logger.info("Size: " + newfilesJsonArray.size());
-                    logger.info("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
+                    logger.fine("Size: " + newfilesJsonArray.size());
+                    logger.fine("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
                     JsonArrayBuilder jsonDataSecondAPI = Json.createArrayBuilder();
 
                     for (JsonObject fileJsonObject : filesJsonArray.getValuesAs(JsonObject.class)) {
@@ -1227,198 +1109,8 @@ public String calculatemime(String fileName) throws InterruptedException {
 
         return finalType;
     }
-    /*
-     * public boolean globusFinishTransfer(Dataset dataset, AuthenticatedUser user)
-     * throws MalformedURLException {
-     * 
-     * logger.info("=====Tasklist == dataset id :" + dataset.getId()); String
-     * directory = null;
-     * 
-     * try {
-     * 
-     * List<FileMetadata> fileMetadatas = new ArrayList<>();
-     * 
-     * StorageIO<Dataset> datasetSIO = DataAccess.getStorageIO(dataset);
-     * 
-     * 
-     * 
-     * DatasetVersion workingVersion = dataset.getEditVersion();
-     * 
-     * if (workingVersion.getCreateTime() != null) {
-     * workingVersion.setCreateTime(new Timestamp(new Date().getTime())); }
-     * 
-     * directory = dataset.getAuthorityForFileStorage() + "/" +
-     * dataset.getIdentifierForFileStorage();
-     * 
-     * System.out.println("======= directory ==== " + directory +
-     * " ====  datasetId :" + dataset.getId()); Map<String, Integer> checksumMapOld
-     * = new HashMap<>();
-     * 
-     * Iterator<FileMetadata> fmIt = workingVersion.getFileMetadatas().iterator();
-     * 
-     * while (fmIt.hasNext()) { FileMetadata fm = fmIt.next(); if (fm.getDataFile()
-     * != null && fm.getDataFile().getId() != null) { String chksum =
-     * fm.getDataFile().getChecksumValue(); if (chksum != null) {
-     * checksumMapOld.put(chksum, 1); } } }
-     * 
-     * List<DataFile> dFileList = new ArrayList<>(); boolean update = false; for
-     * (S3ObjectSummary s3ObjectSummary : datasetSIO.listAuxObjects("")) {
-     * 
-     * String s3ObjectKey = s3ObjectSummary.getKey();
-     * 
-     * 
-     * String t = s3ObjectKey.replace(directory, "");
-     * 
-     * if (t.indexOf(".") > 0) { long totalSize = s3ObjectSummary.getSize(); String
-     * filePath = s3ObjectKey; String fileName =
-     * filePath.split("/")[filePath.split("/").length - 1]; String fullPath =
-     * datasetSIO.getStorageLocation() + "/" + fileName;
-     * 
-     * logger.info("Full path " + fullPath); StorageIO<DvObject> dataFileStorageIO =
-     * DataAccess.getDirectStorageIO(fullPath); InputStream in =
-     * dataFileStorageIO.getInputStream();
-     * 
-     * String checksumVal = FileUtil.calculateChecksum(in,
-     * DataFile.ChecksumType.MD5); //String checksumVal = s3ObjectSummary.getETag();
-     * logger.info("The checksum is " + checksumVal); if
-     * ((checksumMapOld.get(checksumVal) != null)) { logger.info("datasetId :" +
-     * dataset.getId() + "======= filename ==== " + filePath +
-     * " == file already exists "); } else if (filePath.contains("cached") ||
-     * filePath.contains(".thumb")) { logger.info(filePath + " is ignored"); } else
-     * { update = true; logger.info("datasetId :" + dataset.getId() +
-     * "======= filename ==== " + filePath + " == new file   "); try {
-     * 
-     * DataFile datafile = new DataFile(DataFileServiceBean.MIME_TYPE_GLOBUS_FILE);
-     * //MIME_TYPE_GLOBUS datafile.setModificationTime(new Timestamp(new
-     * Date().getTime())); datafile.setCreateDate(new Timestamp(new
-     * Date().getTime())); datafile.setPermissionModificationTime(new Timestamp(new
-     * Date().getTime()));
-     * 
-     * FileMetadata fmd = new FileMetadata();
-     * 
-     * 
-     * fmd.setLabel(fileName); fmd.setDirectoryLabel(filePath.replace(directory,
-     * "").replace(File.separator + fileName, ""));
-     * 
-     * fmd.setDataFile(datafile);
-     * 
-     * datafile.getFileMetadatas().add(fmd);
-     * 
-     * FileUtil.generateS3PackageStorageIdentifierForGlobus(datafile);
-     * logger.info("====  datasetId :" + dataset.getId() + "======= filename ==== "
-     * + filePath + " == added to datafile, filemetadata   ");
-     * 
-     * try { // We persist "SHA1" rather than "SHA-1".
-     * //datafile.setChecksumType(DataFile.ChecksumType.SHA1);
-     * datafile.setChecksumType(DataFile.ChecksumType.MD5);
-     * datafile.setChecksumValue(checksumVal); } catch (Exception cksumEx) {
-     * logger.info("====  datasetId :" + dataset.getId() +
-     * "======Could not calculate  checksumType signature for the new file "); }
-     * 
-     * datafile.setFilesize(totalSize);
-     * 
-     * dFileList.add(datafile);
-     * 
-     * } catch (Exception ioex) { logger.info("datasetId :" + dataset.getId() +
-     * "======Failed to process and/or save the file " + ioex.getMessage()); return
-     * false;
-     * 
-     * } } } } if (update) {
-     * 
-     * List<DataFile> filesAdded = new ArrayList<>();
-     * 
-     * if (dFileList != null && dFileList.size() > 0) {
-     * 
-     * // Dataset dataset = version.getDataset();
-     * 
-     * for (DataFile dataFile : dFileList) {
-     * 
-     * if (dataFile.getOwner() == null) { dataFile.setOwner(dataset);
-     * 
-     * workingVersion.getFileMetadatas().add(dataFile.getFileMetadata());
-     * dataFile.getFileMetadata().setDatasetVersion(workingVersion);
-     * dataset.getFiles().add(dataFile);
-     * 
-     * }
-     * 
-     * filesAdded.add(dataFile);
-     * 
-     * }
-     * 
-     * logger.info("====  datasetId :" + dataset.getId() +
-     * " ===== Done! Finished saving new files to the dataset."); }
-     * 
-     * fileMetadatas.clear(); for (DataFile addedFile : filesAdded) {
-     * fileMetadatas.add(addedFile.getFileMetadata()); } filesAdded = null;
-     * 
-     * if (workingVersion.isDraft()) {
-     * 
-     * logger.info("Async: ====  datasetId :" + dataset.getId() +
-     * " ==== inside draft version ");
-     * 
-     * Timestamp updateTime = new Timestamp(new Date().getTime());
-     * 
-     * workingVersion.setLastUpdateTime(updateTime);
-     * dataset.setModificationTime(updateTime);
-     * 
-     * 
-     * for (FileMetadata fileMetadata : fileMetadatas) {
-     * 
-     * if (fileMetadata.getDataFile().getCreateDate() == null) {
-     * fileMetadata.getDataFile().setCreateDate(updateTime);
-     * fileMetadata.getDataFile().setCreator((AuthenticatedUser) user); }
-     * fileMetadata.getDataFile().setModificationTime(updateTime); }
-     * 
-     * 
-     * } else { logger.info("datasetId :" + dataset.getId() +
-     * " ==== inside released version ");
-     * 
-     * for (int i = 0; i < workingVersion.getFileMetadatas().size(); i++) { for
-     * (FileMetadata fileMetadata : fileMetadatas) { if
-     * (fileMetadata.getDataFile().getStorageIdentifier() != null) {
-     * 
-     * if (fileMetadata.getDataFile().getStorageIdentifier().equals(workingVersion.
-     * getFileMetadatas().get(i).getDataFile().getStorageIdentifier())) {
-     * workingVersion.getFileMetadatas().set(i, fileMetadata); } } } }
-     * 
-     * 
-     * }
-     * 
-     * 
-     * try { Command<Dataset> cmd; logger.info("Async: ====  datasetId :" +
-     * dataset.getId() +
-     * " ======= UpdateDatasetVersionCommand START in globus function "); cmd = new
-     * UpdateDatasetVersionCommand(dataset, new DataverseRequest(user,
-     * (HttpServletRequest) null)); ((UpdateDatasetVersionCommand)
-     * cmd).setValidateLenient(true); //new DataverseRequest(authenticatedUser,
-     * (HttpServletRequest) null) //dvRequestService.getDataverseRequest()
-     * commandEngine.submit(cmd); } catch (CommandException ex) {
-     * logger.log(Level.WARNING, "====  datasetId :" + dataset.getId() +
-     * "======CommandException updating DatasetVersion from batch job: " +
-     * ex.getMessage()); return false; }
-     * 
-     * logger.info("====  datasetId :" + dataset.getId() +
-     * " ======= GLOBUS  CALL COMPLETED SUCCESSFULLY ");
-     * 
-     * //return true; }
-     * 
-     * } catch (Exception e) { String message = e.getMessage();
-     * 
-     * logger.info("====  datasetId :" + dataset.getId() +
-     * " ======= GLOBUS  CALL Exception ============== " + message);
-     * e.printStackTrace(); return false; //return
-     * error(Response.Status.INTERNAL_SERVER_ERROR,
-     * "Uploaded files have passed checksum validation but something went wrong while attempting to move the files into Dataverse. Message was '"
-     * + message + "'."); }
-     * 
-     * String globusBasicToken =
-     * settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusBasicToken, "");
-     * AccessToken clientTokenUser = getClientToken(globusBasicToken);
-     * updatePermision(clientTokenUser, directory, "identity", "r"); return true; }
-     * 
-     */
 
-    GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
+    private GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         Dataset dataset = null;
         if (dvObject instanceof Dataset) {
             dataset = (Dataset) dvObject;
@@ -1435,8 +1127,6 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
         if (GlobusAccessibleStore.isDataverseManaged(driverId) && (dataset != null)) {
             directoryPath = directoryPath + "/" + dataset.getAuthorityForFileStorage() + "/"
                     + dataset.getIdentifierForFileStorage();
-            logger.info("directoryPath now: " + directoryPath);
-
         } else {
             // remote store - may have path in file storageidentifier
             String relPath = dvObject.getStorageIdentifier()
@@ -1446,17 +1136,16 @@ GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
                 directoryPath = directoryPath + relPath.substring(0, filenameStart);
             }
         }
-        logger.info("directoryPath finally: " + directoryPath);
+        logger.fine("directoryPath finally: " + directoryPath);
 
         String endpointId = GlobusAccessibleStore.getTransferEndpointId(driverId);
 
-        logger.info("endpointId: " + endpointId);
+        logger.fine("endpointId: " + endpointId);
 
         String globusToken = GlobusAccessibleStore.getGlobusToken(driverId);
 
         AccessToken accessToken = GlobusServiceBean.getClientToken(globusToken);
         String clientToken = accessToken.getOtherTokens().get(0).getAccessToken();
-        logger.info("clientToken: " + clientToken);
         endpoint = new GlobusEndpoint(endpointId, clientToken, directoryPath);
 
         return endpoint;
@@ -1484,7 +1173,7 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse,
         
         DataFile df = guestbookResponse.getDataFile();
         if (df != null) {
-            logger.info("Single datafile case for writeGuestbookAndStartTransfer");
+            logger.fine("Single datafile case for writeGuestbookAndStartTransfer");
             List<DataFile> downloadDFList = new ArrayList<DataFile>(1);
             downloadDFList.add(df);
             if (!doNotSaveGuestbookResponse) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index c9038047611..96a56d09c0b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -50,7 +50,7 @@ public enum JvmSettings {
     UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"),
     DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"),
     GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"),
-    GLOBUS_RULES_CACHE_MAXAGE(SCOPE_FILES, "globus-rules-cache-maxage"),
+    GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-rules-cache-maxage"),
     FILES(SCOPE_FILES),
     BASE_URL(FILES, "base-url"),
     GLOBUS_TOKEN(FILES, "globus-token"),
diff --git a/src/main/webapp/globus.xhtml b/src/main/webapp/globus.xhtml
deleted file mode 100644
index f4eebd4babf..00000000000
--- a/src/main/webapp/globus.xhtml
+++ /dev/null
@@ -1,30 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:h="http://java.sun.com/jsf/html"
-      xmlns:f="http://java.sun.com/jsf/core"
-      xmlns:ui="http://java.sun.com/jsf/facelets"
-      xmlns:p="http://primefaces.org/ui"
-      xmlns:c="http://xmlns.jcp.org/jsp/jstl/core"
-      xmlns:jsf="http://xmlns.jcp.org/jsf">
-<h:head>
-</h:head>
-<h:body>
-    <ui:composition template="/dataverse_template.xhtml">
-        <ui:define name="body">
-            <script>
-                //<![CDATA[
-                var urlParameters = window.location.search
-                console.log(urlParameters);
-                //]]>
-            </script>
-            <f:metadata>
-                <f:viewParam name="code" value="#{GlobusServiceBean.code}" />
-                <f:viewParam name="state" value="#{GlobusServiceBean.state}" />
-                <f:viewAction action="#{GlobusServiceBean.onLoad}" />
-            </f:metadata>
-
-        </ui:define>
-    </ui:composition>
-
-</h:body>
-</html>

From caa6e684390bb4c36dff45f1de94837f8b632f57 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 4 Dec 2023 18:29:22 -0500
Subject: [PATCH 863/922] revert unrelated changes, old settings

---
 .../harvest/server/web/servlet/OAIServlet.java    | 15 ++++++++++-----
 .../iq/dataverse/settings/JvmSettings.java        |  5 +----
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java
index 19901cae796..96a19acc0e8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java
@@ -73,13 +73,18 @@ public class OAIServlet extends HttpServlet {
     
     @EJB
     SystemConfig systemConfig;
+
+    @Inject
+    @ConfigProperty(name = "dataverse.oai.server.maxidentifiers", defaultValue="100")
+    private Integer maxListIdentifiers;
     
-    //Todo - revert this change - added to get past some local compile issues
-    private Integer maxListIdentifiers=100;
-    
-    private Integer maxListSets=100;
+    @Inject
+    @ConfigProperty(name = "dataverse.oai.server.maxsets", defaultValue="100")
+    private Integer maxListSets;
     
-    private Integer maxListRecords=10;
+    @Inject
+    @ConfigProperty(name = "dataverse.oai.server.maxrecords", defaultValue="10")
+    private Integer maxListRecords;
     
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.web.servlet.OAIServlet");
     // If we are going to stick with this solution - of providing a minimalist 
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
index 96a56d09c0b..fb85ae9adab 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java
@@ -50,10 +50,7 @@ public enum JvmSettings {
     UPLOADS_DIRECTORY(SCOPE_FILES, "uploads"),
     DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"),
     GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"),
-    GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-rules-cache-maxage"),
-    FILES(SCOPE_FILES),
-    BASE_URL(FILES, "base-url"),
-    GLOBUS_TOKEN(FILES, "globus-token"),
+    GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-cache-maxage"),
     
     // SOLR INDEX SETTINGS
     SCOPE_SOLR(PREFIX, "solr"),

From 3babc5aac25710dcc92a90ae861a7b21eef43742 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Mon, 4 Dec 2023 20:35:56 -0500
Subject: [PATCH 864/922] moving the StorageUse member to DvObjectContainer
 from DvObject; moving the em.merge()/em.persist() to the djb. #8549

---
 .../java/edu/harvard/iq/dataverse/DataFile.java | 17 -----------------
 .../iq/dataverse/DataverseServiceBean.java      | 17 ++++++++++++++++-
 .../java/edu/harvard/iq/dataverse/DvObject.java | 14 --------------
 .../harvard/iq/dataverse/DvObjectContainer.java | 14 ++++++++++++--
 .../command/impl/SetCollectionQuotaCommand.java | 15 +--------------
 .../storageuse/StorageUseServiceBean.java       |  1 -
 6 files changed, 29 insertions(+), 49 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
index 2770118d41b..3d8086b142b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java
@@ -641,23 +641,6 @@ public String getFriendlySize() {
         }
     }
     
-    /**
-     * Experimental - record the pre-calculated "storage size" of the file, and 
-     * all its associated auxiliary file objects:
-
-    @Column(nullable = true)
-    private Long storageSize;
-    
-
-    public Long getStorageSize() {
-        return storageSize; 
-    }
-    
-    public void setStorageSize(Long storageSize) {
-        this.storageSize = storageSize; 
-    }
-    * */
-
     public boolean isRestricted() {
         return restricted;
     }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 549b8310122..487215c7a65 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -18,6 +18,7 @@
 import edu.harvard.iq.dataverse.search.IndexServiceBean;
 import edu.harvard.iq.dataverse.search.SolrIndexServiceBean;
 import edu.harvard.iq.dataverse.search.SolrSearchResult;
+import edu.harvard.iq.dataverse.storageuse.StorageQuota;
 import edu.harvard.iq.dataverse.util.StringUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.File;
@@ -919,5 +920,19 @@ public List<Object[]> getDatasetTitlesWithinDataverse(Long dataverseId) {
         return em.createNativeQuery(cqString).getResultList();
     }
 
-    
+    public void saveStorageQuota(Dataverse target, Long allocation) {
+        StorageQuota storageQuota = target.getStorageQuota();
+        
+        if (storageQuota != null) {
+            storageQuota.setAllocation(allocation);
+            em.merge(storageQuota);
+        } else {
+            storageQuota = new StorageQuota(); 
+            storageQuota.setDefinitionPoint(target);
+            storageQuota.setAllocation(allocation);
+            target.setStorageQuota(storageQuota);
+            em.persist(storageQuota);
+        }
+        em.flush();
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObject.java b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
index 515d9f9f153..df249e04663 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObject.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObject.java
@@ -3,7 +3,6 @@
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.pidproviders.PidUtil;
 import edu.harvard.iq.dataverse.storageuse.StorageQuota;
-import edu.harvard.iq.dataverse.storageuse.StorageUse;
 
 import java.sql.Timestamp;
 import java.text.SimpleDateFormat;
@@ -182,10 +181,6 @@ public void setAlternativePersistentIndentifiers(Set<AlternativePersistentIdenti
     @OneToOne(mappedBy = "definitionPoint",cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST}, orphanRemoval=true)
     private StorageQuota storageQuota;
     
-    @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE, CascadeType.PERSIST}, orphanRemoval=true)
-    private StorageUse storageUse;
-    
-    
     public boolean isPreviewImageAvailable() {
         return previewImageAvailable;
     }
@@ -485,13 +480,4 @@ public void setStorageQuota(StorageQuota storageQuota) {
     @OneToMany(mappedBy = "definitionPoint",cascade={ CascadeType.REMOVE, CascadeType.MERGE,CascadeType.PERSIST}, orphanRemoval=true)
     List<RoleAssignment> roleAssignments;
     
-    /**
-     * Should only be used in constructors for DvObjectContainers (Datasets and 
-     * Collections), to make sure new entries are created and persisted in the 
-     * database StorageUse table for every DvObject container we create.
-     * @param storageUse 
-     */
-    public void setStorageUse(StorageUse storageUse) {
-        this.storageUse = storageUse;
-    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
index 2f391e394fa..82057315fbb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DvObjectContainer.java
@@ -2,11 +2,9 @@
 
 import edu.harvard.iq.dataverse.dataaccess.DataAccess;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
-import edu.harvard.iq.dataverse.storageuse.StorageQuota;
 import edu.harvard.iq.dataverse.storageuse.StorageUse;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import jakarta.persistence.CascadeType;
-import java.util.Locale;
 import java.util.Optional;
 
 import jakarta.persistence.MappedSuperclass;
@@ -45,6 +43,9 @@ public boolean isEffectivelyPermissionRoot() {
     
     private Boolean guestbookAtRequest = null;
    
+    @OneToOne(mappedBy = "dvObjectContainer",cascade={ CascadeType.REMOVE, CascadeType.PERSIST}, orphanRemoval=true)
+    private StorageUse storageUse;
+    
     public String getEffectiveStorageDriverId() {
         String id = storageDriver;
         if (StringUtils.isBlank(id)) {
@@ -165,4 +166,13 @@ public void setCurationLabelSetName(String setName) {
         this.externalLabelSetName = setName;
     }
     
+    /**
+     * Should only be used in constructors for DvObjectContainers (Datasets and 
+     * Collections), to make sure new entries are created and persisted in the 
+     * database StorageUse table for every DvObject container we create.
+     * @param storageUse 
+     */
+    public void setStorageUse(StorageUse storageUse) {
+        this.storageUse = storageUse;
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
index cf8fb6fd42e..e52c47a5e7d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/SetCollectionQuotaCommand.java
@@ -9,7 +9,6 @@
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
-import edu.harvard.iq.dataverse.storageuse.StorageQuota;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import java.util.logging.Logger;
 
@@ -49,18 +48,6 @@ public void executeImpl(CommandContext ctxt) throws CommandException {
             throw new IllegalCommandException("Must specify valid allocation in bytes", this);
         }
         
-        StorageQuota storageQuota = dataverse.getStorageQuota();
-        
-        if (storageQuota != null) {
-            storageQuota.setAllocation(allocation);
-            ctxt.em().merge(storageQuota);
-        } else {
-            storageQuota = new StorageQuota(); 
-            storageQuota.setDefinitionPoint(dataverse);
-            storageQuota.setAllocation(allocation);
-            dataverse.setStorageQuota(storageQuota);
-            ctxt.em().persist(storageQuota);
-        }
-        ctxt.em().flush();
+        ctxt.dataverses().saveStorageQuota(dataverse, allocation);
     }    
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
index 18e4ef49640..fbaaff22dee 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
@@ -1,6 +1,5 @@
 package edu.harvard.iq.dataverse.storageuse;
 
-import edu.harvard.iq.dataverse.DvObjectContainer;
 import edu.harvard.iq.dataverse.settings.JvmSettings;
 import jakarta.ejb.Stateless;
 import jakarta.ejb.TransactionAttribute;

From dfa2dc3853254bc8c58bedbfd288a63bcfa07b32 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 04:38:46 -0500
Subject: [PATCH 865/922] remove adaptation for quotas PR that was itself
 changed

---
 .../impl/CreateNewDataFilesCommand.java       | 24 ++-----------------
 1 file changed, 2 insertions(+), 22 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 269ba47643b..0470f59b861 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -3,20 +3,18 @@
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.DatasetVersion;
 import edu.harvard.iq.dataverse.authorization.Permission;
-import edu.harvard.iq.dataverse.dataaccess.DataAccess;
-import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
 import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
 import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
 import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
 import edu.harvard.iq.dataverse.engine.command.CommandContext;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+//import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
 import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
 import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
 import edu.harvard.iq.dataverse.DataFileServiceBean.UserStorageQuota;
 import edu.harvard.iq.dataverse.Dataverse;
-import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.FileUtil;
@@ -85,7 +83,7 @@ public class CreateNewDataFilesCommand extends AbstractCommand<CreateDataFileRes
     private final String newStorageIdentifier; 
     private final String newCheckSum; 
     private DataFile.ChecksumType newCheckSumType;
-    private Long newFileSize;
+    private final Long newFileSize;
 
     public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UserStorageQuota quota, String newCheckSum) {
         this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, null);
@@ -641,24 +639,6 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
                 }
                 logger.fine("Supplied type: " + suppliedContentType + ", finalType: " + finalType);
             }
-            if (newFileSize == null) {
-                // For direct/out-of-band upload, get the size from the underlying service
-                StorageIO<DvObject> sio;
-                try {
-                    sio = DataAccess.getDirectStorageIO(DataAccess.getLocationFromStorageId(newStorageIdentifier, version.getDataset()));
-
-                    // get file size
-                    // Note - some stores (e.g. AWS S3) only offer eventual consistency and a call
-                    // to get the size immediately after uploading may fail. As of the addition of
-                    // PR#9409 adding storage quotas, we are now requiring size to be available
-                    // earlier. If this is seen, adding
-                    // a delay/retry may help
-                    newFileSize = sio.retrieveSizeFromMedia();
-                } catch (IOException e) {
-                    // If we don't get a file size, a CommandExecutionException will be thrown later in the code
-                    e.printStackTrace();
-                }
-            }
         }
         // Finally, if none of the special cases above were applicable (or 
         // if we were unable to unpack an uploaded file, etc.), we'll just 

From c78613e60ca7a2442753d6382b0ace3c7fd07316 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 5 Dec 2023 08:42:23 -0500
Subject: [PATCH 866/922] one more refinement for the flyway script. #8549

---
 .../storageuse/StorageUseServiceBean.java     | 33 ++++++++++---------
 .../V6.0.0.5__8549-collection-quotas.sql      | 13 ++++++++
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
index fbaaff22dee..7aea7a7b596 100644
--- a/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/storageuse/StorageUseServiceBean.java
@@ -46,23 +46,24 @@ public Long findStorageSizeByDvContainerId(Long dvObjectId) {
      */
     @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
     public void incrementStorageSizeRecursively(Long dvObjectContainerId, Long increment) {
-        //@todo should throw exceptions if either parameter is null
-        Optional<Boolean> allow = JvmSettings.STORAGEUSE_DISABLE_UPDATES.lookupOptional(Boolean.class);
-        if (!(allow.isPresent() && allow.get())) {
-            String queryString = "WITH RECURSIVE uptree (id, owner_id) AS\n"
-                    + "("
-                    + "    SELECT id, owner_id\n"
-                    + "    FROM dvobject\n"
-                    + "    WHERE id=" + dvObjectContainerId + "\n"
-                    + "    UNION ALL\n"
-                    + "    SELECT dvobject.id, dvobject.owner_id\n"
-                    + "    FROM dvobject\n"
-                    + "    JOIN uptree ON dvobject.id = uptree.owner_id)\n"
-                    + "UPDATE storageuse SET sizeinbytes=COALESCE(sizeinbytes,0)+" + increment + "\n"
-                    + "FROM uptree\n"
-                    + "WHERE dvobjectcontainer_id = uptree.id;";
+        if (dvObjectContainerId != null && increment != null) {
+            Optional<Boolean> allow = JvmSettings.STORAGEUSE_DISABLE_UPDATES.lookupOptional(Boolean.class);
+            if (!(allow.isPresent() && allow.get())) {
+                String queryString = "WITH RECURSIVE uptree (id, owner_id) AS\n"
+                        + "("
+                        + "    SELECT id, owner_id\n"
+                        + "    FROM dvobject\n"
+                        + "    WHERE id=" + dvObjectContainerId + "\n"
+                        + "    UNION ALL\n"
+                        + "    SELECT dvobject.id, dvobject.owner_id\n"
+                        + "    FROM dvobject\n"
+                        + "    JOIN uptree ON dvobject.id = uptree.owner_id)\n"
+                        + "UPDATE storageuse SET sizeinbytes=COALESCE(sizeinbytes,0)+" + increment + "\n"
+                        + "FROM uptree\n"
+                        + "WHERE dvobjectcontainer_id = uptree.id;";
 
-            int parentsUpdated = em.createNativeQuery(queryString).executeUpdate();
+                int parentsUpdated = em.createNativeQuery(queryString).executeUpdate();
+            }
         }
         // @todo throw an exception if the number of parent dvobjects updated by
         // the query is < 2 - ? 
diff --git a/src/main/resources/db/migration/V6.0.0.5__8549-collection-quotas.sql b/src/main/resources/db/migration/V6.0.0.5__8549-collection-quotas.sql
index 3657642c267..d6c067056ec 100644
--- a/src/main/resources/db/migration/V6.0.0.5__8549-collection-quotas.sql
+++ b/src/main/resources/db/migration/V6.0.0.5__8549-collection-quotas.sql
@@ -38,6 +38,19 @@ AND fileobject.id = file.id
 AND dt.datafile_id = file.id
 GROUP BY datasetobject.id) o, dataset ds WHERE o.id = dvobject.id AND dvobject.dtype='Dataset' AND dvobject.id = ds.id AND ds.harvestingclient_id IS null;
 
+-- there may also be some auxiliary files registered in the database, such as
+-- the content generated and deposited by external tools - diff. privacy stats
+-- being one of the example. These are also considered the "payload" files that
+-- we want to count for the purposes of calculating storage use.
+UPDATE dvobject SET tempStorageSize=tempStorageSize+o.combinedStorageSize
+FROM (SELECT datasetobject.id, COALESCE(SUM(aux.fileSize),0) AS combinedStorageSize
+FROM dvobject fileobject, dvobject datasetobject, datafile file, auxiliaryFile aux
+WHERE fileobject.owner_id = datasetobject.id
+AND fileobject.id = file.id
+AND aux.datafile_id = file.id
+GROUP BY datasetobject.id) o, dataset ds WHERE o.id = dvobject.id AND dvobject.dtype='Dataset' AND dvobject.id = ds.id AND ds.harvestingclient_id IS null;
+
+
 -- ... and then we can repeat the same for collections, by setting the storage size
 -- to the sum of the storage sizes of the datasets *directly* in each collection:
 -- (no attemp is made yet to recursively count the sizes all the chilld sub-collections)

From 0c02b15aab711acbfb7f2c957c4482313b3997b9 Mon Sep 17 00:00:00 2001
From: Jim Myers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 09:50:33 -0500
Subject: [PATCH 867/922] try QDR /logo endpoint

---
 .../edu/harvard/iq/dataverse/api/Datasets.java   | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index af6059cf882..828ba218cc4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -1971,6 +1971,22 @@ public Response getDatasetThumbnail(@PathParam("id") String idSupplied) {
         }
     }
 
+    @GET
+    @Produces({ "image/png" })
+    @Path("{id}/logo")
+    public Response getDatasetLogo(@PathParam("id") String idSupplied) {
+        try {
+            Dataset dataset = findDatasetOrDie(idSupplied);
+            InputStream is = DatasetUtil.getLogoAsInputStream(dataset);
+            if (is == null) {
+                return notFound("Logo not available");
+            }
+            return Response.ok(is).build();
+        } catch (WrappedResponse wr) {
+            return notFound("Logo not available");
+        }
+    }
+
     // TODO: Rather than only supporting looking up files by their database IDs (dataFileIdSupplied), consider supporting persistent identifiers.
     @POST
     @AuthRequired

From 8c9f1242d53aea5ecc906bd4a2a3f5d12a884224 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 5 Dec 2023 10:13:53 -0500
Subject: [PATCH 868/922] switch minio to creds jenkins expects #6783

---
 docker-compose-dev.yml                                   | 9 ++++-----
 .../java/edu/harvard/iq/dataverse/api/S3AccessIT.java    | 4 ++--
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 6bc50f7e764..98376e255dd 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -41,8 +41,8 @@ services:
         -Ddataverse.files.minio1.path-style-access=true
         -Ddataverse.files.minio1.upload-redirect=false
         -Ddataverse.files.minio1.download-redirect=false
-        -Ddataverse.files.minio1.access-key=minioadmin
-        -Ddataverse.files.minio1.secret-key=minioadmin
+        -Ddataverse.files.minio1.access-key=4cc355_k3y
+        -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k35
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4848:4848" # HTTP (Payara Admin Console)
@@ -211,9 +211,8 @@ services:
     volumes:
       - minio_storage:/data
     environment:
-      # these are the defaults but are here for clarity
-      MINIO_ROOT_USER: minioadmin
-      MINIO_ROOT_PASSWORD: minioadmin
+      MINIO_ROOT_USER: 4cc355_k3y
+      MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k35
     command: server /data
 
 networks:
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index f5e4ce6a794..daf04bb3d14 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -54,8 +54,8 @@ public static void setUp() {
                 .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyLocalStack, secretKeyLocalStack)))
                 .withEndpointConfiguration(new EndpointConfiguration("s3.localhost.localstack.cloud:4566", Regions.US_EAST_2.getName())).build();
 
-        String accessKeyMinio = "minioadmin";
-        String secretKeyMinio = "minioadmin";
+        String accessKeyMinio = "4cc355_k3y";
+        String secretKeyMinio = "s3cr3t_4cc355_k35";
         s3minio = AmazonS3ClientBuilder.standard()
                 // https://stackoverflow.com/questions/72205086/amazonss3client-throws-unknownhostexception-if-attempting-to-connect-to-a-local
                 .withPathStyleAccessEnabled(Boolean.TRUE)

From 6a7d8d1c6f76c8e54f9759f643204aa339c5bdd0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 5 Dec 2023 10:33:19 -0500
Subject: [PATCH 869/922] make assertions about users #6783

---
 .../java/edu/harvard/iq/dataverse/api/S3AccessIT.java     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index daf04bb3d14..7c1531cbfaf 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -99,9 +99,10 @@ public void testNonDirectUpload() {
         String driverLabel = "MinIO";
 
         Response createSuperuser = UtilIT.createRandomUser();
+        createSuperuser.then().assertThat().statusCode(200);
         String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
         String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
-        UtilIT.makeSuperUser(superusername);
+        UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200);
         Response storageDrivers = listStorageDrivers(superuserApiToken);
         storageDrivers.prettyPrint();
         // TODO where is "Local/local" coming from?
@@ -118,6 +119,7 @@ public void testNonDirectUpload() {
 
         //create user who will make a dataverse/dataset
         Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(200);
         String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
 
@@ -208,9 +210,10 @@ public void testDirectUpload() {
         String driverId = "localstack1";
         String driverLabel = "LocalStack";
         Response createSuperuser = UtilIT.createRandomUser();
+        createSuperuser.then().assertThat().statusCode(200);
         String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
         String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
-        UtilIT.makeSuperUser(superusername);
+        UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200);
         Response storageDrivers = listStorageDrivers(superuserApiToken);
         storageDrivers.prettyPrint();
         // TODO where is "Local/local" coming from?
@@ -227,6 +230,7 @@ public void testDirectUpload() {
 
         //create user who will make a dataverse/dataset
         Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(200);
         String username = UtilIT.getUsernameFromResponse(createUser);
         String apiToken = UtilIT.getApiTokenFromResponse(createUser);
 

From b9f48913e498ec96ef8f5994c21e7bb549e747e0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Tue, 5 Dec 2023 10:41:45 -0500
Subject: [PATCH 870/922] move methods to UtilIT #6783

---
 .../harvard/iq/dataverse/api/S3AccessIT.java  | 75 +++----------------
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 50 +++++++++++++
 2 files changed, 62 insertions(+), 63 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index 7c1531cbfaf..1306c30d9c1 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -103,7 +103,7 @@ public void testNonDirectUpload() {
         String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
         String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
         UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200);
-        Response storageDrivers = listStorageDrivers(superuserApiToken);
+        Response storageDrivers = UtilIT.listStorageDrivers(superuserApiToken);
         storageDrivers.prettyPrint();
         // TODO where is "Local/local" coming from?
         String drivers = """
@@ -127,18 +127,18 @@ public void testNonDirectUpload() {
         createDataverseResponse.prettyPrint();
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
-        Response originalStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken);
         originalStorageDriver.prettyPrint();
         originalStorageDriver.then().assertThat()
                 .body("data.message", equalTo("undefined"))
                 .statusCode(200);
 
-        Response setStorageDriverToS3 = setStorageDriver(dataverseAlias, driverLabel, superuserApiToken);
+        Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken);
         setStorageDriverToS3.prettyPrint();
         setStorageDriverToS3.then().assertThat()
                 .statusCode(200);
 
-        Response updatedStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        Response updatedStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken);
         updatedStorageDriver.prettyPrint();
         updatedStorageDriver.then().assertThat()
                 .statusCode(200);
@@ -214,7 +214,7 @@ public void testDirectUpload() {
         String superuserApiToken = UtilIT.getApiTokenFromResponse(createSuperuser);
         String superusername = UtilIT.getUsernameFromResponse(createSuperuser);
         UtilIT.makeSuperUser(superusername).then().assertThat().statusCode(200);
-        Response storageDrivers = listStorageDrivers(superuserApiToken);
+        Response storageDrivers = UtilIT.listStorageDrivers(superuserApiToken);
         storageDrivers.prettyPrint();
         // TODO where is "Local/local" coming from?
         String drivers = """
@@ -238,18 +238,18 @@ public void testDirectUpload() {
         createDataverseResponse.prettyPrint();
         String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
 
-        Response originalStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken);
         originalStorageDriver.prettyPrint();
         originalStorageDriver.then().assertThat()
                 .body("data.message", equalTo("undefined"))
                 .statusCode(200);
 
-        Response setStorageDriverToS3 = setStorageDriver(dataverseAlias, driverLabel, superuserApiToken);
+        Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken);
         setStorageDriverToS3.prettyPrint();
         setStorageDriverToS3.then().assertThat()
                 .statusCode(200);
 
-        Response updatedStorageDriver = getStorageDriver(dataverseAlias, superuserApiToken);
+        Response updatedStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken);
         updatedStorageDriver.prettyPrint();
         updatedStorageDriver.then().assertThat()
                 .statusCode(200);
@@ -275,7 +275,7 @@ public void testDirectUpload() {
 //
 //        String fileId = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.id");
         long size = 1000000000l;
-        Response getUploadUrls = getUploadUrls(datasetPid, size, apiToken);
+        Response getUploadUrls = UtilIT.getUploadUrls(datasetPid, size, apiToken);
         getUploadUrls.prettyPrint();
         getUploadUrls.then().assertThat().statusCode(200);
 
@@ -298,7 +298,7 @@ public void testDirectUpload() {
         String contentsOfFile = "foobar";
 
         InputStream inputStream = new ByteArrayInputStream(contentsOfFile.getBytes(StandardCharsets.UTF_8));
-        Response uploadFileDirect = uploadFileDirect(localhostUrl, inputStream);
+        Response uploadFileDirect = UtilIT.uploadFileDirect(localhostUrl, inputStream);
         uploadFileDirect.prettyPrint();
         /*
         Direct upload to MinIO is failing with errors like this:
@@ -357,7 +357,7 @@ public void testDirectUpload() {
         assertEquals(contentsOfFile, s3Object);
 
         System.out.println("direct download...");
-        Response getHeaders = downloadFileNoRedirect(Integer.valueOf(fileId), apiToken);
+        Response getHeaders = UtilIT.downloadFileNoRedirect(Integer.valueOf(fileId), apiToken);
         for (Header header : getHeaders.getHeaders()) {
             System.out.println("direct download header: " + header);
         }
@@ -371,7 +371,7 @@ public void testDirectUpload() {
         } catch (UnsupportedEncodingException ex) {
         }
 
-        Response downloadFile = downloadFromUrl(decodedDownloadUrl);
+        Response downloadFile = UtilIT.downloadFromUrl(decodedDownloadUrl);
         downloadFile.prettyPrint();
         downloadFile.then().assertThat().statusCode(200);
 
@@ -394,55 +394,4 @@ public void testDirectUpload() {
 
     }
 
-    //TODO: move these into UtilIT. They are here for now to avoid merge conflicts
-    static Response listStorageDrivers(String apiToken) {
-        return given()
-                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/admin/dataverse/storageDrivers");
-    }
-
-    static Response getStorageDriver(String dvAlias, String apiToken) {
-        return given()
-                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
-                .get("/api/admin/dataverse/" + dvAlias + "/storageDriver");
-    }
-
-    static Response setStorageDriver(String dvAlias, String label, String apiToken) {
-        return given()
-                .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken)
-                .body(label)
-                .put("/api/admin/dataverse/" + dvAlias + "/storageDriver");
-    }
-
-    static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes, String apiToken) {
-        String idInPath = idOrPersistentIdOfDataset; // Assume it's a number.
-        String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
-        if (!NumberUtils.isCreatable(idOrPersistentIdOfDataset)) {
-            idInPath = ":persistentId";
-            optionalQueryParam = "&persistentId=" + idOrPersistentIdOfDataset;
-        }
-        RequestSpecification requestSpecification = given();
-        if (apiToken != null) {
-            requestSpecification = given()
-                    .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken);
-        }
-        return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam);
-    }
-
-    static Response uploadFileDirect(String url, InputStream inputStream) {
-        return given()
-                .header("x-amz-tagging", "dv-state=temp")
-                .body(inputStream)
-                .put(url);
-    }
-
-    static Response downloadFileNoRedirect(Integer fileId, String apiToken) {
-        return given().when().redirects().follow(false)
-                .get("/api/access/datafile/" + fileId + "?key=" + apiToken);
-    }
-
-    static Response downloadFromUrl(String url) {
-        return given().get(url);
-    }
-
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 9b264086c27..12bb069424f 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -2361,6 +2361,56 @@ static Response deleteStorageSite(long storageSiteId) {
                 .delete("/api/admin/storageSites/" + storageSiteId);
     }
 
+    static Response listStorageDrivers(String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/admin/dataverse/storageDrivers");
+    }
+
+    static Response getStorageDriver(String dvAlias, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .get("/api/admin/dataverse/" + dvAlias + "/storageDriver");
+    }
+
+    static Response setStorageDriver(String dvAlias, String label, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(label)
+                .put("/api/admin/dataverse/" + dvAlias + "/storageDriver");
+    }
+
+    static Response getUploadUrls(String idOrPersistentIdOfDataset, long sizeInBytes, String apiToken) {
+        String idInPath = idOrPersistentIdOfDataset; // Assume it's a number.
+        String optionalQueryParam = ""; // If idOrPersistentId is a number we'll just put it in the path.
+        if (!NumberUtils.isCreatable(idOrPersistentIdOfDataset)) {
+            idInPath = ":persistentId";
+            optionalQueryParam = "&persistentId=" + idOrPersistentIdOfDataset;
+        }
+        RequestSpecification requestSpecification = given();
+        if (apiToken != null) {
+            requestSpecification = given()
+                    .header(API_TOKEN_HTTP_HEADER, apiToken);
+        }
+        return requestSpecification.get("/api/datasets/" + idInPath + "/uploadurls?size=" + sizeInBytes + optionalQueryParam);
+    }
+
+    static Response uploadFileDirect(String url, InputStream inputStream) {
+        return given()
+                .header("x-amz-tagging", "dv-state=temp")
+                .body(inputStream)
+                .put(url);
+    }
+
+    static Response downloadFileNoRedirect(Integer fileId, String apiToken) {
+        return given().when().redirects().follow(false)
+                .get("/api/access/datafile/" + fileId + "?key=" + apiToken);
+    }
+
+    static Response downloadFromUrl(String url) {
+        return given().get(url);
+    }
+
     static Response metricsDataversesToMonth(String yyyymm, String queryParams) {
         String optionalYyyyMm = "";
         if (yyyymm != null) {

From 7349ed9f754e05ff7b16a24ea8f3c24c060ed593 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 10:43:38 -0500
Subject: [PATCH 871/922] get logo, picking 48px size for datafile thumbs

FWIW: QDR generates a 400px version here and then uses styling
 to fit the page. Not sure what the motivation for that was without
 digging.
---
 .../iq/dataverse/dataset/DatasetUtil.java     | 63 +++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
index 096f1f87acc..ccf861ebdc8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java
@@ -411,6 +411,69 @@ public static InputStream getThumbnailAsInputStream(Dataset dataset, int size) {
             return nonDefaultDatasetThumbnail;
         }
     }
+    
+    public static InputStream getLogoAsInputStream(Dataset dataset) {
+        if (dataset == null) {
+            return null;
+        }
+        StorageIO<Dataset> dataAccess = null;
+
+        try {
+            dataAccess = DataAccess.getStorageIO(dataset);
+        } catch (IOException ioex) {
+            logger.warning("getLogo(): Failed to initialize dataset StorageIO for " + dataset.getStorageIdentifier()
+                    + " (" + ioex.getMessage() + ")");
+        }
+
+        InputStream in = null;
+        try {
+            if (dataAccess == null) {
+                logger.warning(
+                        "getLogo(): Failed to initialize dataset StorageIO for " + dataset.getStorageIdentifier());
+            } else {
+                in = dataAccess.getAuxFileAsInputStream(datasetLogoFilenameFinal);
+            }
+        } catch (IOException ex) {
+            logger.fine(
+                    "Dataset-level thumbnail file does not exist, or failed to open; will try to find an image file that can be used as the thumbnail.");
+        }
+
+        if (in == null) {
+            DataFile thumbnailFile = dataset.getThumbnailFile();
+
+            if (thumbnailFile == null) {
+                if (dataset.isUseGenericThumbnail()) {
+                    logger.fine("Dataset (id :" + dataset.getId() + ") does not have a logo and is 'Use Generic'.");
+                    return null;
+                } else {
+                    thumbnailFile = attemptToAutomaticallySelectThumbnailFromDataFiles(dataset, null);
+                    if (thumbnailFile == null) {
+                        logger.fine("Dataset (id :" + dataset.getId()
+                                + ") does not have a logo available that could be selected automatically.");
+                        return null;
+                    } else {
+
+                    }
+                }
+            }
+            if (thumbnailFile.isRestricted()) {
+                logger.fine("Dataset (id :" + dataset.getId()
+                        + ") has a logo the user selected but the file must have later been restricted. Returning null.");
+                return null;
+            }
+
+            try {
+                in = ImageThumbConverter.getImageThumbnailAsInputStream(thumbnailFile.getStorageIO(),
+                        ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE).getInputStream();
+            } catch (IOException ioex) {
+                logger.warning("getLogo(): Failed to get logo from DataFile for " + dataset.getStorageIdentifier()
+                        + " (" + ioex.getMessage() + ")");
+                ioex.printStackTrace();
+            }
+
+        }
+        return in;
+    }
 
     /**
      * The dataset logo is the file that a user uploads which is *not* one of

From c194d74b2029917de050fe5d40b237b23bddf3ab Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 5 Dec 2023 10:59:46 -0500
Subject: [PATCH 872/922] Clarified the sentence about the initial deployment
 in the release note. #8549

---
 doc/release-notes/8549-collection-quotas.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/8549-collection-quotas.md b/doc/release-notes/8549-collection-quotas.md
index 29b84213cfb..b3635d0c5a1 100644
--- a/doc/release-notes/8549-collection-quotas.md
+++ b/doc/release-notes/8549-collection-quotas.md
@@ -1,3 +1,3 @@
 This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
-Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the deployment.
+Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of 6.1
 

From cf7e664e626994419ca3a1c80785290da7efe683 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 5 Dec 2023 12:02:41 -0500
Subject: [PATCH 873/922] moved the entitymanager calls from a command to the
 service #8549

---
 .../edu/harvard/iq/dataverse/DataverseServiceBean.java    | 8 ++++++++
 .../engine/command/impl/DeleteCollectionQuotaCommand.java | 4 +---
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
index 487215c7a65..b6e666e8058 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java
@@ -935,4 +935,12 @@ public void saveStorageQuota(Dataverse target, Long allocation) {
         }
         em.flush();
     }
+    
+    public void disableStorageQuota(StorageQuota storageQuota) {
+        if (storageQuota != null && storageQuota.getAllocation() != null) {
+            storageQuota.setAllocation(null);
+            em.merge(storageQuota);
+            em.flush();
+        }
+    }
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
index 4015228366b..c0f863686da 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DeleteCollectionQuotaCommand.java
@@ -46,9 +46,7 @@ public void executeImpl(CommandContext ctxt) throws CommandException {
         StorageQuota storageQuota = targetDataverse.getStorageQuota();
         
         if (storageQuota != null && storageQuota.getAllocation() != null) {
-            storageQuota.setAllocation(null);
-            ctxt.em().merge(storageQuota);
-            ctxt.em().flush();
+            ctxt.dataverses().disableStorageQuota(storageQuota);
         } 
         // ... and if no quota was enabled on the collection - nothing to do = success
     }    

From dfa49c3720f866f36df0b6cd712f1c5144dfee44 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 14:31:32 -0500
Subject: [PATCH 874/922] rename flyway script

---
 ...thumb-failures.sql => V6.0.0.6__9506-track-thumb-failures.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/main/resources/db/migration/{V6.0.0.5__9506-track-thumb-failures.sql => V6.0.0.6__9506-track-thumb-failures.sql} (100%)

diff --git a/src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql b/src/main/resources/db/migration/V6.0.0.6__9506-track-thumb-failures.sql
similarity index 100%
rename from src/main/resources/db/migration/V6.0.0.5__9506-track-thumb-failures.sql
rename to src/main/resources/db/migration/V6.0.0.6__9506-track-thumb-failures.sql

From 70a3442cc9a6c672ef8a553be8b279b3b8ea1b52 Mon Sep 17 00:00:00 2001
From: Leonid Andreev <leonid@hmdc.harvard.edu>
Date: Tue, 5 Dec 2023 14:36:21 -0500
Subject: [PATCH 875/922] updated aux. file service bean #8549

---
 .../dataverse/AuxiliaryFileServiceBean.java   | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java
index 8c96f98ce39..363622ba3bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/AuxiliaryFileServiceBean.java
@@ -2,6 +2,7 @@
 package edu.harvard.iq.dataverse;
 
 import edu.harvard.iq.dataverse.dataaccess.StorageIO;
+import edu.harvard.iq.dataverse.storageuse.StorageUseServiceBean;
 import edu.harvard.iq.dataverse.util.FileUtil;
 import edu.harvard.iq.dataverse.util.SystemConfig;
 
@@ -46,6 +47,8 @@ public class AuxiliaryFileServiceBean implements java.io.Serializable {
     @EJB
     private SystemConfig systemConfig;
     
+    @EJB
+    StorageUseServiceBean storageUseService; 
 
     public AuxiliaryFile find(Object pk) {
         return em.find(AuxiliaryFile.class, pk);
@@ -126,6 +129,13 @@ public AuxiliaryFile processAuxiliaryFile(InputStream fileInputStream, DataFile
                 }
                 dataFile.getAuxiliaryFiles().add(auxFile);
             }
+            // We've just added this file to storage; increment the StorageUse
+            // record if needed. 
+            if (auxFile.getFileSize() != null 
+                    && auxFile.getFileSize() > 0 
+                    && dataFile.getOwner() != null ) {
+                storageUseService.incrementStorageSizeRecursively(dataFile.getOwner().getId(), auxFile.getFileSize());
+            }
         } catch (IOException ioex) {
             logger.severe("IO Exception trying to save auxiliary file: " + ioex.getMessage());
             throw new InternalServerErrorException();
@@ -181,6 +191,7 @@ public void deleteAuxiliaryFile(DataFile dataFile, String formatTag, String form
         if (af == null) {
             throw new FileNotFoundException();
         }
+        Long auxFileSize = af.getFileSize();
         em.remove(af);
         StorageIO<?> storageIO;
         storageIO = dataFile.getStorageIO();
@@ -188,6 +199,14 @@ public void deleteAuxiliaryFile(DataFile dataFile, String formatTag, String form
         if (storageIO.isAuxObjectCached(auxExtension)) {
             storageIO.deleteAuxObject(auxExtension);
         }
+        // We've just deleted this file from storage; update the StorageUse
+        // record if needed. 
+        if (auxFileSize != null
+                && auxFileSize > 0
+                && dataFile.getOwner() != null) {
+            storageUseService.incrementStorageSizeRecursively(dataFile.getOwner().getId(), (0L - auxFileSize));
+        }
+        
     }
 
     public List<AuxiliaryFile> findAuxiliaryFiles(DataFile dataFile) {

From c54a85fca9377b74efc0e74e8a70a6de2f6fccc4 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 5 Dec 2023 14:52:23 -0500
Subject: [PATCH 876/922] #9464 add caveats to release note.

---
 doc/release-notes/9464-json-validation.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/9464-json-validation.md b/doc/release-notes/9464-json-validation.md
index 4b08f2ca9dd..f104263ba35 100644
--- a/doc/release-notes/9464-json-validation.md
+++ b/doc/release-notes/9464-json-validation.md
@@ -1,3 +1,3 @@
-Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. (Issue #9464 and #9465)
+Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release funtionality is limited to json format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
 
 For documentation see the API changelog: http://preview.guides.gdcc.io/en/develop/api/changelog.html

From 2379828c2737260901b23020a436f5cab6cc962a Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Tue, 5 Dec 2023 15:05:12 -0500
Subject: [PATCH 877/922] Update native-api.rst

---
 doc/sphinx-guides/source/api/native-api.rst | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 2d37c3b07ae..29aa7c880ac 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -510,7 +510,9 @@ The fully expanded example above (without environment variables) looks like this
 Retrieve a Dataset JSON Schema for a Collection
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Retrieves a JSON schema customized for a given collection in order to validate a dataset JSON file prior to creating the dataset:
+Retrieves a JSON schema customized for a given collection in order to validate a dataset JSON file prior to creating the dataset. This
+first version of the schema only includes required elements and fields. In the future we plan to improve the schema by adding controlled
+vocabulary and more robust dataset field format testing:
 
 .. code-block:: bash
 
@@ -535,7 +537,8 @@ While it is recommended to download a copy of the JSON Schema from the collectio
 Validate Dataset JSON File for a Collection
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Validates a dataset JSON file customized for a given collection prior to creating the dataset:
+Validates a dataset JSON file customized for a given collection prior to creating the dataset. The validation only tests for json formatting
+and the presence of required elements:
 
 .. code-block:: bash
 

From dd2d9726e3125975493fa6dbf70578d76fa5f07c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 16:47:04 -0500
Subject: [PATCH 878/922] globus store options

---
 .../source/installation/config.rst            | 50 +++++++++++++++++--
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 7b32da8f6c3..e0e4d4cd89e 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -499,14 +499,14 @@ Logging & Slow Performance
 
 .. _file-storage:
 
-File Storage: Using a Local Filesystem and/or Swift and/or Object Stores and/or Trusted Remote Stores
------------------------------------------------------------------------------------------------------
+File Storage: Using a Local Filesystem and/or Swift and/or Object Stores and/or Trusted Remote Stores and/or Globus Stores
+--------------------------------------------------------------------------------------------------------------------------
 
 By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara6/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.\<id\>.directory`` JVM option described below.
 
-A Dataverse installation can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-Dataverse collection basis.
+A Dataverse installation can alternately store files in a Swift or S3-compatible object store, or on a Globus endpoint, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-Dataverse collection basis.
 
-A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
+A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web or Globus accessible trusted remote store.
 
 A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\<id\>.upload-out-of-band`` JVM option to ``true``.
 By default, Dataverse supports uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
@@ -958,7 +958,7 @@ Once you have configured a trusted remote store, you can point your users to the
     dataverse.files.<id>.type                    ``remote``          **Required** to mark this storage as remote.                                (none)
     dataverse.files.<id>.label                   <?>                 **Required** label to be shown in the UI for this storage.                  (none)
     dataverse.files.<id>.base-url                <?>                 **Required** All files must have URLs of the form <baseUrl>/* .             (none)
-    dataverse.files.<id>.base-store              <?>                 **Optional** The id of a base store (of type file, s3, or swift).           (the default store)
+    dataverse.files.<id>.base-store              <?>                 **Required** The id of a base store (of type file, s3, or swift).           (the default store)
     dataverse.files.<id>.download-redirect       ``true``/``false``  Enable direct download (should usually be true).                            ``false``
     dataverse.files.<id>.secret-key               <?>                 A key used to sign download requests sent to the remote store. Optional.   (none)
     dataverse.files.<id>.url-expiration-minutes  <?>                 If direct downloads and using signing: time until links expire. Optional.   60
@@ -967,6 +967,46 @@ Once you have configured a trusted remote store, you can point your users to the
     
     ===========================================  ==================  ==========================================================================  ===================
 
+.. _globus-storage:
+
+Globus Storage
+++++++++++++++
+
+Globus stores allow Dataverse to manage files stored in Globus endpoints or to reference files in remote Globus endpoints, with users leveraging Globus to transfer files to/from Dataverse (rather than using HTTP/HTTPS).
+See :doc:`/developers/big-data-support` for additional information on how to use a globus store. Consult the `Globus documentation <https://docs.globus.org/>`_ for information about using Globus and configuring Globus endpoints.
+
+In addition to having the type "globus" and requiring a label, Globus Stores share many options with Trusted Remote Stores and options to specify and access a Globus endpoint(s). As with Remote Stores, Globus Stores also use a baseStore - a file, s3, or swift store that can be used to store additional ancillary dataset files (e.g. metadata exports, thumbnails, auxiliary files, etc.).
+These and other available options are described in the table below.
+
+There are two types of Globus stores
+- managed - where Dataverse manages the Globus endpoint, deciding where transferred files are stored and managing access control for users transferring files to/from Dataverse
+- remote - where Dataverse references files that remain on trusted remote Globus endpoints
+
+For managed stores, there are two variants, connecting to standard/file-based Globus endpoints and to endpoints using an underlying S3 store via the Globus S3 Connector.
+With the former, Dataverse has no direct access to the file contents and functionality related to ingest, fixity hash validation, etc. are not available. With the latter, Dataverse can access files internally via S3 and the functionality supported is similar to that when using S3 direct upload. 
+
+Once you have configured a globus store, it is recommended that you install the `dataverse-globus app <https://github.com/scholarsportal/dataverse-globus>`_ to allow transfers in/out of Dataverse to be initated via the Dataverse user interface. Alternately, you can point your users to the :doc:`/developers/globus-api` for information about API support.
+
+.. table::
+    :align: left
+
+    =======================================================  ==================  ==========================================================================  ===================
+    JVM Option                                               Value               Description                                                                 Default value
+    =======================================================  ==================  ==========================================================================  ===================
+    dataverse.files.<id>.type                                ``globus``          **Required** to mark this storage as globus enabled.                        (none)
+    dataverse.files.<id>.label                               <?>                 **Required** label to be shown in the UI for this storage.                  (none)
+    dataverse.files.<id>.base-store                          <?>                 **Required** The id of a base store (of type file, s3, or swift).           (the default store)
+    dataverse.files.<id>.remote-store-name                   <?>                 A short name used in the UI to indicate where a file is located. Optional.  (none)
+    dataverse.files.<id>.remote-store-url                    <?>                 A url to an info page about the remote store used in the UI. Optional.      (none)
+    dataverse.files.<id>.managed                             ``true``/``false``  Whether dataverse manages an associated Globus endpoint                     ``false``
+    dataverse.files.<id>.transfer-endpoint-with-basepath     <?>                 The *managed* Globus endpoint id and associated base path for file storage  (none)
+    dataverse.files.<id>.globus-token                        <?>                 A Globus token (base64 endcoded <Globus user id>:<Credential> 
+                                                                                 for a managed store) - using a microprofile alias is recommended            (none)
+    dataverse.files.<id>.reference-endpoints-with-basepaths  <?>                 A comma separated list of *remote* trusted Globus endpoint id/<basePath>s   (none)
+    dataverse.files.<id>.files-not-accessible-by-dataverse   ``true``/``false``  Should be true for S3 Connector-based *managed* stores                      ``false``
+    
+    =======================================================  ==================  ==========================================================================  ===================
+    
 .. _temporary-file-storage:
 
 Temporary Upload File Storage

From 4d7818a7be615033bd00261a6a0951c703c0ad3b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 16:59:04 -0500
Subject: [PATCH 879/922] merge miss

---
 .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index 13ec049fa0a..8afc365417e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -87,7 +87,7 @@
 /* 
     Amazon AWS S3 driver
  */
-public class S3AccessIO<T extends DvObject> extends StorageIO<T> implements GlobusAccessibleStore {
+public class S3AccessIO<T extends DvObject> extends StorageIO<T> {
 
     private static final Config config = ConfigProvider.getConfig();
     private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.S3AccessIO");
@@ -1194,7 +1194,6 @@ private static AmazonS3 getClient(String driverId) {
              * * if a profile and static credentials are both explicitly set, the profile will be used preferentially, and 
              * * if no store-specific credentials are set, the global credentials will be preferred over using any "default" profile credentials that are found.
              */
-            String s3profile = getConfigParamForDriver(driverId, PROFILE,"default");
 
             ArrayList<AWSCredentialsProvider> providers = new ArrayList<>();
 

From ceacf7e92c045a61b96205536f442dc48142cb2a Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Tue, 5 Dec 2023 16:59:40 -0500
Subject: [PATCH 880/922] add a stub globus api page since it is referenced in
 the config doc

---
 .../source/developers/globus-api.rst          | 282 ++++++++++++++++++
 doc/sphinx-guides/source/developers/index.rst |   1 +
 2 files changed, 283 insertions(+)
 create mode 100644 doc/sphinx-guides/source/developers/globus-api.rst

diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst
new file mode 100644
index 00000000000..2775ffd2142
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/globus-api.rst
@@ -0,0 +1,282 @@
+Globus Transfer API
+===================
+
+The Globus API addresses three use cases:
+* Transfer to a Dataverse-managed Globus endpoint (File-based or using the Globus S3 Connector)
+* Reference of files that will remain in a remote Globus endpoint
+* Transfer from a Dataverse-managed Globus endpoint
+
+The ability for Dataverse to interact with Globus endpoints is configured via 
+Direct upload involves a series of three activities, each involving interacting with the server for a Dataverse installation:
+
+* Requesting initiation of a transfer from the server
+* Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3
+* A call to the server to register the file/files as part of the dataset/replace a file in the dataset or to cancel the transfer
+
+This API is only enabled when a Dataset is configured with a data store supporting direct S3 upload.
+Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in the Dataverse installation (e.g. should be considered temporary and deleted.)
+
+ 
+Requesting Direct Upload of a DataFile
+--------------------------------------
+To initiate a transfer of a file to S3, make a call to the Dataverse installation indicating the size of the file to upload. The response will include a pre-signed URL(s) that allow the client to transfer the file. Pre-signed URLs include a short-lived token authorizing the action represented by the URL.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+  export SIZE=1000000000
+ 
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/uploadurls?persistentId=$PERSISTENT_IDENTIFIER&size=$SIZE"
+
+The response to this call, assuming direct uploads are enabled, will be one of two forms:
+
+Single URL: when the file is smaller than the size at which uploads must be broken into multiple parts
+
+.. code-block:: bash
+
+  {
+    "status":"OK",
+    "data":{
+      "url":"...",
+      "partSize":1073741824,
+      "storageIdentifier":"s3://demo-dataverse-bucket:177883619b8-892ca9f7112e"
+    }
+  }
+
+Multiple URLs: when the file must be uploaded in multiple parts. The part size is set by the Dataverse installation and, for AWS-based storage, range from 5 MB to 5 GB
+
+.. code-block:: bash
+
+  {
+    "status":"OK",
+    "data":{
+    "urls":{
+      "1":"...",
+      "2":"...",
+      "3":"...",
+      "4":"...",
+      "5":"..."
+    }
+    "abort":"/api/datasets/mpupload?...",
+    "complete":"/api/datasets/mpupload?..."
+    "partSize":1073741824,
+    "storageIdentifier":"s3://demo-dataverse-bucket:177883b000e-49cedef268ac"
+  }
+
+In the example responses above, the URLs, which are very long, have been omitted. These URLs reference the S3 server and the specific object identifier that will be used, starting with, for example, https://demo-dataverse-bucket.s3.amazonaws.com/10.5072/FK2FOQPJS/177883b000e-49cedef268ac?...
+
+The client must then use the URL(s) to PUT the file, or if the file is larger than the specified partSize, parts of the file. 
+
+In the single part case, only one call to the supplied URL is required:
+
+.. code-block:: bash
+
+    curl -H 'x-amz-tagging:dv-state=temp' -X PUT -T <filename> "<supplied url>"
+
+
+In the multipart case, the client must send each part and collect the 'eTag' responses from the server. The calls for this are the same as the one for the single part case except that each call should send a <partSize> slice of the total file, with the last part containing the remaining bytes.
+The responses from the S3 server for these calls will include the 'eTag' for the uploaded part. 
+
+To successfully conclude the multipart upload, the client must call the 'complete' URI, sending a json object including the part eTags:
+
+.. code-block:: bash
+
+    curl -X PUT "$SERVER_URL/api/datasets/mpload?..." -d '{"1":"<eTag1 string>","2":"<eTag2 string>","3":"<eTag3 string>","4":"<eTag4 string>","5":"<eTag5 string>"}'
+  
+If the client is unable to complete the multipart upload, it should call the abort URL:
+
+.. code-block:: bash
+  
+    curl -X DELETE "$SERVER_URL/api/datasets/mpload?..."
+   
+  
+.. _direct-add-to-dataset-api:
+
+Adding the Uploaded file to the Dataset
+---------------------------------------
+
+Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to a Dataverse installation but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter.
+jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for:
+
+* "storageIdentifier" - String, as specified in prior calls
+* "fileName" - String
+* "mimeType" - String
+* fixity/checksum: either: 
+
+  * "md5Hash" - String with MD5 hash value, or
+  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings 
+
+The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+  export JSON_DATA="{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}"
+
+  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
+  
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+
+To add multiple Uploaded Files to the Dataset
+---------------------------------------------
+
+Once the files exists in the s3 bucket, a final API call is needed to add all the files to the Dataset. In this API call, additional metadata is added using the "jsonData" parameter.
+jsonData for this call is an array of objects that normally include information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for:
+
+* "description" - A description of the file
+* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset
+* "storageIdentifier" - String
+* "fileName" - String
+* "mimeType" - String
+* "fixity/checksum" either:
+
+  * "md5Hash" - String with MD5 hash value, or
+  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings
+
+The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV
+  export JSON_DATA="[{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}, \
+                      {'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53', 'fileName':'file2.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123789'}}]"
+
+  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
+
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+
+
+Replacing an existing file in the Dataset
+-----------------------------------------
+
+Once the file exists in the s3 bucket, a final API call is needed to register it as a replacement of an existing file. This call is the same call used to replace a file to a Dataverse installation but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter.
+jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, whether to allow the mimetype to change (forceReplace=true), etc. For direct uploads, the jsonData object must include values for:
+
+* "storageIdentifier" - String, as specified in prior calls
+* "fileName" - String
+* "mimeType" - String
+* fixity/checksum: either: 
+
+  * "md5Hash" - String with MD5 hash value, or
+  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings 
+
+The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512.
+Note that the API call does not validate that the file matches the hash value supplied. If a Dataverse instance is configured to validate file fixity hashes at publication time, a mismatch would be caught at that time and cause publication to fail.
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export FILE_IDENTIFIER=5072
+  export JSON_DATA='{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "forceReplace":"true", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}'
+
+  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/files/$FILE_IDENTIFIER/replace" -F "jsonData=$JSON_DATA"
+  
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+
+Replacing multiple existing files in the Dataset
+------------------------------------------------
+
+Once the replacement files exist in the s3 bucket, a final API call is needed to register them as replacements for existing files. In this API call, additional metadata is added using the "jsonData" parameter.
+jsonData for this call is array of objects that normally include information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must include some additional values:
+
+* "fileToReplaceId" - the id of the file being replaced
+* "forceReplace" - whether to replace a file with one of a different mimetype (optional, default is false)
+* "description" - A description of the file
+* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset
+* "storageIdentifier" - String
+* "fileName" - String
+* "mimeType" - String
+* "fixity/checksum" either:
+
+  * "md5Hash" - String with MD5 hash value, or
+  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings
+
+
+The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512
+
+.. code-block:: bash
+
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV
+  export JSON_DATA='[{"fileToReplaceId": 10, "description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}},{"fileToReplaceId": 11, "forceReplace": true, "description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]'
+
+  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/replaceFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
+
+The JSON object returned as a response from this API call includes a "data" that indicates how many of the file replacements succeeded and provides per-file error messages for those that don't, e.g.
+
+.. code-block::
+
+  {
+    "status": "OK",
+    "data": {
+      "Files": [
+        {
+          "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42",
+          "errorMessage": "Bad Request:The file to replace does not belong to this dataset.",
+          "fileDetails": {
+            "fileToReplaceId": 10,
+            "description": "My description.",
+            "directoryLabel": "data/subdir1",
+            "categories": [
+              "Data"
+            ],
+            "restrict": "false",
+            "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42",
+            "fileName": "file1.Bin",
+            "mimeType": "application/octet-stream",
+            "checksum": {
+              "@type": "SHA-1",
+              "@value": "123456"
+            }
+          }
+        },
+        {
+          "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53",
+          "successMessage": "Replaced successfully in the dataset",
+          "fileDetails": {
+            "description": "My description.",
+            "label": "file2.txt",
+            "restricted": false,
+            "directoryLabel": "data/subdir1",
+            "categories": [
+              "Data"
+            ],
+            "dataFile": {
+              "persistentId": "",
+              "pidURL": "",
+              "filename": "file2.txt",
+              "contentType": "text/plain",
+              "filesize": 2407,
+              "description": "My description.",
+              "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53",
+              "rootDataFileId": 11,
+              "previousDataFileId": 11,
+              "checksum": {
+                "type": "SHA-1",
+                "value": "123789"
+              }
+            }
+          }
+        }
+      ],
+      "Result": {
+        "Total number of files": 2,
+        "Number of files successfully replaced": 1
+      }
+    }
+  }
+
+
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst
index 60d97feeef9..458a78a6c95 100755
--- a/doc/sphinx-guides/source/developers/index.rst
+++ b/doc/sphinx-guides/source/developers/index.rst
@@ -39,6 +39,7 @@ Developer Guide
    big-data-support
    aux-file-support
    s3-direct-upload-api
+   globus-api
    dataset-semantic-metadata-api
    dataset-migration-api 
    workflows

From 811d79a7f8d017745fcfd782b233ec583d3669e2 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Dec 2023 08:33:38 -0500
Subject: [PATCH 881/922] change minio access key, more l33t #6783

---
 docker-compose-dev.yml                                     | 2 +-
 src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 98376e255dd..e68215d53d2 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -212,7 +212,7 @@ services:
       - minio_storage:/data
     environment:
       MINIO_ROOT_USER: 4cc355_k3y
-      MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k35
+      MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k3y
     command: server /data
 
 networks:
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index 1306c30d9c1..41446349093 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -55,7 +55,7 @@ public static void setUp() {
                 .withEndpointConfiguration(new EndpointConfiguration("s3.localhost.localstack.cloud:4566", Regions.US_EAST_2.getName())).build();
 
         String accessKeyMinio = "4cc355_k3y";
-        String secretKeyMinio = "s3cr3t_4cc355_k35";
+        String secretKeyMinio = "s3cr3t_4cc355_k3y";
         s3minio = AmazonS3ClientBuilder.standard()
                 // https://stackoverflow.com/questions/72205086/amazonss3client-throws-unknownhostexception-if-attempting-to-connect-to-a-local
                 .withPathStyleAccessEnabled(Boolean.TRUE)

From a81ad72a0896073e043ee57848e571d7a3754a8a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Dec 2023 10:50:46 -0500
Subject: [PATCH 882/922] comment out optional listing of buckets #6783

---
 .../harvard/iq/dataverse/api/S3AccessIT.java  | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
index 41446349093..74150ca120a 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java
@@ -62,16 +62,15 @@ public static void setUp() {
                 .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyMinio, secretKeyMinio)))
                 .withEndpointConfiguration(new EndpointConfiguration("http://localhost:9000", Regions.US_EAST_1.getName())).build();
 
-        System.out.println("buckets on LocalStack before attempting to create " + BUCKET_NAME);
-        for (Bucket bucket : s3localstack.listBuckets()) {
-            System.out.println("bucket: " + bucket);
-        }
-
-        System.out.println("buckets on MinIO before attempting to create " + BUCKET_NAME);
-        for (Bucket bucket : s3minio.listBuckets()) {
-            System.out.println("bucket: " + bucket);
-        }
-
+//        System.out.println("buckets on LocalStack before attempting to create " + BUCKET_NAME);
+//        for (Bucket bucket : s3localstack.listBuckets()) {
+//            System.out.println("bucket: " + bucket);
+//        }
+//
+//        System.out.println("buckets on MinIO before attempting to create " + BUCKET_NAME);
+//        for (Bucket bucket : s3minio.listBuckets()) {
+//            System.out.println("bucket: " + bucket);
+//        }
         // create bucket if it doesn't exist
         // Note that we create the localstack bucket with conf/localstack/buckets.sh
         // because we haven't figured out how to create it properly in Java.

From 0bd9f139e5dca2851ca88ed12c5e31af9c5bbfe9 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Wed, 6 Dec 2023 11:01:04 -0500
Subject: [PATCH 883/922] Update doc/release-notes/6.1-release-notes.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/release-notes/6.1-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index c2b52ab34b8..06a3e01f7af 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -1,6 +1,6 @@
 # Dataverse 6.1
 
-(If this note appears truncated on the GitHub Releases page, you can view it in full in the source tree: https://github.com/IQSS/dataverse/blob/master/doc/release-notes/6.1-release-notes.md)
+Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.1 rather than the list of releases, which will cut them off.
 
 This release brings new features, enhancements, and bug fixes to the Dataverse software.
 Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.

From c97d7b55e2932dacaa19e4e3ac403c88a25bd2ee Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 11:01:56 -0500
Subject: [PATCH 884/922] globus api doc

---
 .../source/developers/globus-api.rst          | 348 ++++++++----------
 1 file changed, 149 insertions(+), 199 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst
index 2775ffd2142..6a94f220dc2 100644
--- a/doc/sphinx-guides/source/developers/globus-api.rst
+++ b/doc/sphinx-guides/source/developers/globus-api.rst
@@ -6,277 +6,227 @@ The Globus API addresses three use cases:
 * Reference of files that will remain in a remote Globus endpoint
 * Transfer from a Dataverse-managed Globus endpoint
 
-The ability for Dataverse to interact with Globus endpoints is configured via 
-Direct upload involves a series of three activities, each involving interacting with the server for a Dataverse installation:
+The ability for Dataverse to interact with Globus endpoints is configured via a Globus store - see :ref:`globus-storage`.
 
-* Requesting initiation of a transfer from the server
-* Use of the pre-signed URL(s) returned in that call to perform an upload/multipart-upload of the file to S3
-* A call to the server to register the file/files as part of the dataset/replace a file in the dataset or to cancel the transfer
+Globus transfers (or referencing a remote endpoint) for upload and download transfers involve a series of steps. These can be accomplished using the Dataverse and Globus APIs. (These are used internally by the `dataverse-globus app <https://github.com/scholarsportal/dataverse-globus>`_ when transfers are done via the Dataverse UI.) 
 
-This API is only enabled when a Dataset is configured with a data store supporting direct S3 upload.
-Administrators should be aware that partial transfers, where a client starts uploading the file/parts of the file and does not contact the server to complete/cancel the transfer, will result in data stored in S3 that is not referenced in the Dataverse installation (e.g. should be considered temporary and deleted.)
+Requesting Upload or Download Parameters
+----------------------------------------
 
- 
-Requesting Direct Upload of a DataFile
---------------------------------------
-To initiate a transfer of a file to S3, make a call to the Dataverse installation indicating the size of the file to upload. The response will include a pre-signed URL(s) that allow the client to transfer the file. Pre-signed URLs include a short-lived token authorizing the action represented by the URL.
+The first step in preparing for a Globus transfer/reference operation is to request the parameters relevant for a given dataset:
 
 .. code-block:: bash
 
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
-  export SIZE=1000000000
- 
-  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/uploadurls?persistentId=$PERSISTENT_IDENTIFIER&size=$SIZE"
-
-The response to this call, assuming direct uploads are enabled, will be one of two forms:
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/globusUploadParameters?locale=$LOCALE"
 
-Single URL: when the file is smaller than the size at which uploads must be broken into multiple parts
+The response will be of the form:
 
 .. code-block:: bash
 
   {
-    "status":"OK",
-    "data":{
-      "url":"...",
-      "partSize":1073741824,
-      "storageIdentifier":"s3://demo-dataverse-bucket:177883619b8-892ca9f7112e"
+          "status": "OK",
+          "data": {
+              "queryParameters": {
+                  "datasetId": 29,
+                  "siteUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com",
+                  "datasetVersion": ":draft",
+                  "dvLocale": "en",
+                  "datasetPid": "doi:10.5072/FK2/ILLPXE",
+                  "managed": "true",
+                  "endpoint": "d8c42580-6528-4605-9ad8-116a61982644"
+              },
+              "signedUrls": [
+                  {
+                      "name": "requestGlobusTransferPaths",
+                      "httpMethod": "POST",
+                      "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/requestGlobusUploadPaths?until=2023-11-22T01:52:03.648&user=dataverseAdmin&method=POST&token=63ac4bb748d12078dded1074916508e19e6f6b61f64294d38e0b528010b07d48783cf2e975d7a1cb6d4a3c535f209b981c7c6858bc63afdfc0f8ecc8a139b44a",
+                      "timeOut": 300
+                  },
+                  {
+                      "name": "addGlobusFiles",
+                      "httpMethod": "POST",
+                      "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/addGlobusFiles?until=2023-11-22T01:52:03.648&user=dataverseAdmin&method=POST&token=2aaa03f6b9f851a72e112acf584ffc0758ed0cc8d749c5a6f8c20494bb7bc13197ab123e1933f3dde2711f13b347c05e6cec1809a8f0b5484982570198564025",
+                      "timeOut": 300
+                  },
+                  {
+                      "name": "getDatasetMetadata",
+                      "httpMethod": "GET",
+                      "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/versions/:draft?until=2023-11-22T01:52:03.649&user=dataverseAdmin&method=GET&token=1878d6a829cd5540e89c07bdaf647f1bea5314cc7a55433b0b506350dd330cad61ade3714a8ee199a7b464fb3b8cddaea0f32a89ac3bfc4a86cd2ea3004ecbb8",
+                      "timeOut": 300
+                  },
+                  {
+                      "name": "getFileListing",
+                      "httpMethod": "GET",
+                      "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/versions/:draft/files?until=2023-11-22T01:52:03.650&user=dataverseAdmin&method=GET&token=78e8ca8321624f42602af659227998374ef3788d0feb43d696a0e19086e0f2b3b66b96981903a1565e836416c504b6248cd3c6f7c2644566979bd16e23a99622",
+                      "timeOut": 300
+                  }
+              ]
+          }
     }
-  }
 
-Multiple URLs: when the file must be uploaded in multiple parts. The part size is set by the Dataverse installation and, for AWS-based storage, range from 5 MB to 5 GB
+The response includes the id for the Globus endpoint to use along with several signed URLs.
 
-.. code-block:: bash
+The getDatasetMetadata and getFileListing URLs are just signed versions of the standard Dataset metadata and file listing API calls. The other two are Globus specific.
 
-  {
-    "status":"OK",
-    "data":{
-    "urls":{
-      "1":"...",
-      "2":"...",
-      "3":"...",
-      "4":"...",
-      "5":"..."
-    }
-    "abort":"/api/datasets/mpupload?...",
-    "complete":"/api/datasets/mpupload?..."
-    "partSize":1073741824,
-    "storageIdentifier":"s3://demo-dataverse-bucket:177883b000e-49cedef268ac"
-  }
+If called for a dataset using a store that is configured with a remote Globus endpoint(s), the return response is similar but the response includes a
+the "managed" parameter will be false, the "endpoint" parameter is replaced with a JSON array of "referenceEndpointsWithPaths" and the
+requestGlobusTransferPaths and addGlobusFiles URLs are replaced with ones for requestGlobusReferencePaths and addFiles. All of these calls are
+describe further below.
+
+The call to set up for a transfer out (download) is similar:
 
-In the example responses above, the URLs, which are very long, have been omitted. These URLs reference the S3 server and the specific object identifier that will be used, starting with, for example, https://demo-dataverse-bucket.s3.amazonaws.com/10.5072/FK2FOQPJS/177883b000e-49cedef268ac?...
+.. code-block:: bash
 
-The client must then use the URL(s) to PUT the file, or if the file is larger than the specified partSize, parts of the file. 
+  curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/globusDownloadParameters?locale=$LOCALE"
 
-In the single part case, only one call to the supplied URL is required:
+Note that this API call supports an additional downloadId query parameter. This is only used when the globus-dataverse app is called from the Dataverse user interface. There is no need to use it when calling the API directly.
 
-.. code-block:: bash
+The returned response includes the same getDatasetMetadata and getFileListing URLs as in the upload case and includes "monitorGlobusDownload" and "requestGlobusDownload" URLs. The response will also indicate whether the store is "managed" and will provide the "endpoint" from which downloads can be made.
 
-    curl -H 'x-amz-tagging:dv-state=temp' -X PUT -T <filename> "<supplied url>"
 
+Performing an Upload/Transfer In
+--------------------------------
 
-In the multipart case, the client must send each part and collect the 'eTag' responses from the server. The calls for this are the same as the one for the single part case except that each call should send a <partSize> slice of the total file, with the last part containing the remaining bytes.
-The responses from the S3 server for these calls will include the 'eTag' for the uploaded part. 
+The information from the API call above can be used to provide a user with information about the dataset and to prepare to transfer or to reference files (based on the "managed" parameter). 
 
-To successfully conclude the multipart upload, the client must call the 'complete' URI, sending a json object including the part eTags:
+Once the user identifies which files are to be added, the requestGlobusTransferPaths or requestGlobusReferencePaths URLs can be called. These both reference the same API call but must be used with different entries in the JSON body sent:
 
 .. code-block:: bash
 
-    curl -X PUT "$SERVER_URL/api/datasets/mpload?..." -d '{"1":"<eTag1 string>","2":"<eTag2 string>","3":"<eTag3 string>","4":"<eTag4 string>","5":"<eTag5 string>"}'
-  
-If the client is unable to complete the multipart upload, it should call the abort URL:
+  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+  export SERVER_URL=https://demo.dataverse.org
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+  export LOCALE=en-US
+ 
+  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/datasets/:persistentId/requestGlobusUpload"
 
-.. code-block:: bash
-  
-    curl -X DELETE "$SERVER_URL/api/datasets/mpload?..."
-   
+Note that when using the dataverse-globus app or the return from the previous call, the URL for this call will be signed and no API_TOKEN is needed. 
   
-.. _direct-add-to-dataset-api:
+In the managed case, the JSON body sent must include the id of the Globus user that will perform the transfer and the number of files that will be transferred:
 
-Adding the Uploaded file to the Dataset
----------------------------------------
+.. code-block:: bash
+  {
+    "principal":"d15d4244-fc10-47f3-a790-85bdb6db9a75", 
+    "numberOfFiles":2
+  }
 
-Once the file exists in the s3 bucket, a final API call is needed to add it to the Dataset. This call is the same call used to upload a file to a Dataverse installation but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter.
-jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for:
+In the remote reference case, the JSON body sent must include the Globus endpoint/paths that will be referenced:
 
-* "storageIdentifier" - String, as specified in prior calls
-* "fileName" - String
-* "mimeType" - String
-* fixity/checksum: either: 
+.. code-block:: bash
+  {
+    "referencedFiles":[
+      "d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt"
+    ]
+  }
+    
+The response will include a JSON object. In the managed case, the map is from new assigned file storageidentifiers and specific paths on the managed Globus endpoint:
+.. code-block:: bash
 
-  * "md5Hash" - String with MD5 hash value, or
-  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings 
+  {
+    "status":"OK",
+    "data":{
+      "globusm://18b49d3688c-62137dcb06e4":"/hdc1/10.5072/FK2/ILLPXE/18b49d3688c-62137dcb06e4",
+      "globusm://18b49d3688c-5c17d575e820":"/hdc1/10.5072/FK2/ILLPXE/18b49d3688c-5c17d575e820"
+    }
+  }
 
-The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512
+In the managed case, the specified Globus principal is granted write permission to the specified endpoint/path,
+which will allow initiation of a transfer from the external endpoint to the managed endpoint using the Globus API.
+The permission will be revoked if the transfer is not started and the next call to Dataverse to finish the transfer are not made within a short time (configurable, default of 5 minutes).
+ 
+In the remote/reference case, the map is from the initially supplied endpoint/paths to the new assigned file storageidentifiers:
 
 .. code-block:: bash
 
-  export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-  export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
-  export JSON_DATA="{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}"
-
-  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
-  
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
-
-To add multiple Uploaded Files to the Dataset
----------------------------------------------
+  {
+    "status":"OK",
+    "data":{
+      "d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt":"globus://18bf8c933f4-ed2661e7d19b//d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt"
+    }
+  }
 
-Once the files exists in the s3 bucket, a final API call is needed to add all the files to the Dataset. In this API call, additional metadata is added using the "jsonData" parameter.
-jsonData for this call is an array of objects that normally include information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must also include values for:
 
-* "description" - A description of the file
-* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset
-* "storageIdentifier" - String
-* "fileName" - String
-* "mimeType" - String
-* "fixity/checksum" either:
 
-  * "md5Hash" - String with MD5 hash value, or
-  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings
+Adding Files to the Dataset
+---------------------------
 
-The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512
+In the managed case, once a Globus transfer has been initiated a final API call is made to Dataverse to provide it with the task identifier of the transfer and information about the files being transferred:
 
 .. code-block:: bash
 
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV
-  export JSON_DATA="[{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42', 'fileName':'file1.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}, \
-                      {'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53', 'fileName':'file2.txt', 'mimeType':'text/plain', 'checksum': {'@type': 'SHA-1', '@value': '123789'}}]"
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+  export JSON_DATA="{"taskIdentifier":"3f530302-6c48-11ee-8428-378be0d9c521", \
+                    "files": [{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b3972213f-f6b5c2221423", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "1234"}}, \
+                    {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b39722140-50eb7d3c5ece", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "2345"}}]}"
 
-  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
+  
+  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:multipart/form-data" -X POST "$SERVER_URL/api/datasets/:persistentId/addGlobusFiles -F "jsonData=$JSON_DATA""
 
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+Note that the mimetype is multipart/form-data, matching the /addFiles API call. ALso note that the API_TOKEN is not needed when using a signed URL.
 
+With this information, Dataverse will begin to monitor the transfer and when it completes, will add all files for which the transfer succeeded.
+As the transfer can take significant time and the API call is asynchronous, the only way to determine if the transfer succeeded via API is to use the standard calls to check the dataset lock state and contents.
 
-Replacing an existing file in the Dataset
------------------------------------------
+Once the transfer completes, Dataverse will remove the write permission for the principal.
 
-Once the file exists in the s3 bucket, a final API call is needed to register it as a replacement of an existing file. This call is the same call used to replace a file to a Dataverse installation but, rather than sending the file bytes, additional metadata is added using the "jsonData" parameter.
-jsonData normally includes information such as a file description, tags, provenance, whether the file is restricted, whether to allow the mimetype to change (forceReplace=true), etc. For direct uploads, the jsonData object must include values for:
+Note that when using a managed endpoint that uses the Globus S3 Connector, the checksum should be correct as Dataverse can validate it. For file-based endpoints, the checksum should be included if available but Dataverse cannot verify it.
 
-* "storageIdentifier" - String, as specified in prior calls
-* "fileName" - String
-* "mimeType" - String
-* fixity/checksum: either: 
+In the remote/reference case, where there is no transfer to monitor, the standard /addFiles API call (see :ref:`direct-add-to-dataset-api`) is used instead. There are no changes for the Globus case.
 
-  * "md5Hash" - String with MD5 hash value, or
-  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings 
+Downloading/Transfer Out Via Globus
+-----------------------------------
 
-The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512.
-Note that the API call does not validate that the file matches the hash value supplied. If a Dataverse instance is configured to validate file fixity hashes at publication time, a mismatch would be caught at that time and cause publication to fail.
+To begin downloading files, the requestGlobusDownload URL is used:
 
 .. code-block:: bash
 
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
-  export FILE_IDENTIFIER=5072
-  export JSON_DATA='{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "forceReplace":"true", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}}'
-
-  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/files/$FILE_IDENTIFIER/replace" -F "jsonData=$JSON_DATA"
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
   
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/datasets/:persistentId/requestGlobusDownload"
 
-Replacing multiple existing files in the Dataset
-------------------------------------------------
+The JSON body sent should include a list of file ids to download and, for a managed endpoint, the Globus principal that will make the transfer:
 
-Once the replacement files exist in the s3 bucket, a final API call is needed to register them as replacements for existing files. In this API call, additional metadata is added using the "jsonData" parameter.
-jsonData for this call is array of objects that normally include information such as a file description, tags, provenance, whether the file is restricted, etc. For direct uploads, the jsonData object must include some additional values:
+.. code-block:: bash
+  {
+    "principal":"d15d4244-fc10-47f3-a790-85bdb6db9a75", 
+    "fileIds":[60, 61]
+  }
+  
+Note that this API call takes an optional downloadId parameter that is used with the dataverse-globus app. When downloadId is included, the list of fileIds is not needed.
 
-* "fileToReplaceId" - the id of the file being replaced
-* "forceReplace" - whether to replace a file with one of a different mimetype (optional, default is false)
-* "description" - A description of the file
-* "directoryLabel" - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset
-* "storageIdentifier" - String
-* "fileName" - String
-* "mimeType" - String
-* "fixity/checksum" either:
+The response is a JSON object mapping the requested file Ids to Globus endpoint/paths. In the managed case, the principal will have been given read permissions for the specified paths:
 
-  * "md5Hash" - String with MD5 hash value, or
-  * "checksum" - Json Object with "@type" field specifying the algorithm used and "@value" field with the value from that algorithm, both Strings
+.. code-block:: bash
 
+  {
+    "status":"OK",
+    "data":{
+      "60": "d8c42580-6528-4605-9ad8-116a61982644/hdc1/10.5072/FK2/ILLPXE/18bf3af9c78-92b8e168090e",
+     "61": "d8c42580-6528-4605-9ad8-116a61982644/hdc1/10.5072/FK2/ILLPXE/18bf3af9c78-c8d81569305c"
+    }
+  }
 
-The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.DataFile.CheckSumType class and currently include MD5, SHA-1, SHA-256, and SHA-512
+For the remote case, the use can perform the transfer without further contact with Dataverse. In the managed case, the user must initiate the transfer via the Globus API and then inform Dataverse.
+Dataverse will then monitor the transfer and revoke the read permission when the transfer is complete. (Not making this last call could result in failure of the transfer.)
 
 .. code-block:: bash
 
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
-  export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV
-  export JSON_DATA='[{"fileToReplaceId": 10, "description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123456"}},{"fileToReplaceId": 11, "forceReplace": true, "description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "SHA-1", "@value": "123789"}}]'
-
-  curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/replaceFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
-
-The JSON object returned as a response from this API call includes a "data" that indicates how many of the file replacements succeeded and provides per-file error messages for those that don't, e.g.
+  export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+  
+  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/datasets/:persistentId/monitorGlobusDownload"
+  
+The JSON body sent just contains the task identifier for the transfer:
 
-.. code-block::
+.. code-block:: bash
 
   {
-    "status": "OK",
-    "data": {
-      "Files": [
-        {
-          "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42",
-          "errorMessage": "Bad Request:The file to replace does not belong to this dataset.",
-          "fileDetails": {
-            "fileToReplaceId": 10,
-            "description": "My description.",
-            "directoryLabel": "data/subdir1",
-            "categories": [
-              "Data"
-            ],
-            "restrict": "false",
-            "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357c42",
-            "fileName": "file1.Bin",
-            "mimeType": "application/octet-stream",
-            "checksum": {
-              "@type": "SHA-1",
-              "@value": "123456"
-            }
-          }
-        },
-        {
-          "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53",
-          "successMessage": "Replaced successfully in the dataset",
-          "fileDetails": {
-            "description": "My description.",
-            "label": "file2.txt",
-            "restricted": false,
-            "directoryLabel": "data/subdir1",
-            "categories": [
-              "Data"
-            ],
-            "dataFile": {
-              "persistentId": "",
-              "pidURL": "",
-              "filename": "file2.txt",
-              "contentType": "text/plain",
-              "filesize": 2407,
-              "description": "My description.",
-              "storageIdentifier": "s3://demo-dataverse-bucket:176e28068b0-1c3f80357d53",
-              "rootDataFileId": 11,
-              "previousDataFileId": 11,
-              "checksum": {
-                "type": "SHA-1",
-                "value": "123789"
-              }
-            }
-          }
-        }
-      ],
-      "Result": {
-        "Total number of files": 2,
-        "Number of files successfully replaced": 1
-      }
-    }
+    "taskIdentifier":"b5fd01aa-8963-11ee-83ae-d5484943e99a"
   }
+ 
 
-
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.

From c7d73f64177745fa7892543407025f9130dcb83b Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 11:25:22 -0500
Subject: [PATCH 885/922] default for globus-cache-maxage

---
 src/main/resources/META-INF/microprofile-config.properties | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties
index 504b5e46735..ec8427795ee 100644
--- a/src/main/resources/META-INF/microprofile-config.properties
+++ b/src/main/resources/META-INF/microprofile-config.properties
@@ -18,6 +18,7 @@ dataverse.build=
 dataverse.files.directory=${STORAGE_DIR:/tmp/dataverse}
 dataverse.files.uploads=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/uploads
 dataverse.files.docroot=${STORAGE_DIR:${com.sun.aas.instanceRoot}}/docroot
+dataverse.files.globus-cache-maxage=5
 
 # SEARCH INDEX
 dataverse.solr.host=localhost

From 1fb7ddf6d89a1b36f9a059f016ac617aa6ec3758 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 11:27:42 -0500
Subject: [PATCH 886/922] fix spacing

---
 doc/sphinx-guides/source/developers/globus-api.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst
index 6a94f220dc2..5b2b6982866 100644
--- a/doc/sphinx-guides/source/developers/globus-api.rst
+++ b/doc/sphinx-guides/source/developers/globus-api.rst
@@ -105,6 +105,7 @@ Note that when using the dataverse-globus app or the return from the previous ca
 In the managed case, the JSON body sent must include the id of the Globus user that will perform the transfer and the number of files that will be transferred:
 
 .. code-block:: bash
+
   {
     "principal":"d15d4244-fc10-47f3-a790-85bdb6db9a75", 
     "numberOfFiles":2
@@ -113,6 +114,7 @@ In the managed case, the JSON body sent must include the id of the Globus user t
 In the remote reference case, the JSON body sent must include the Globus endpoint/paths that will be referenced:
 
 .. code-block:: bash
+
   {
     "referencedFiles":[
       "d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt"
@@ -120,6 +122,7 @@ In the remote reference case, the JSON body sent must include the Globus endpoin
   }
     
 The response will include a JSON object. In the managed case, the map is from new assigned file storageidentifiers and specific paths on the managed Globus endpoint:
+
 .. code-block:: bash
 
   {
@@ -161,7 +164,6 @@ In the managed case, once a Globus transfer has been initiated a final API call
                     "files": [{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b3972213f-f6b5c2221423", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "1234"}}, \
                     {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b39722140-50eb7d3c5ece", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "2345"}}]}"
 
-  
   curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:multipart/form-data" -X POST "$SERVER_URL/api/datasets/:persistentId/addGlobusFiles -F "jsonData=$JSON_DATA""
 
 Note that the mimetype is multipart/form-data, matching the /addFiles API call. ALso note that the API_TOKEN is not needed when using a signed URL.
@@ -191,6 +193,7 @@ To begin downloading files, the requestGlobusDownload URL is used:
 The JSON body sent should include a list of file ids to download and, for a managed endpoint, the Globus principal that will make the transfer:
 
 .. code-block:: bash
+
   {
     "principal":"d15d4244-fc10-47f3-a790-85bdb6db9a75", 
     "fileIds":[60, 61]

From 6fee16dec8125390ea6aa7221a19fde0db2b9730 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Wed, 6 Dec 2023 11:52:24 -0500
Subject: [PATCH 887/922] #10151 incorporate json schema

---
 doc/release-notes/6.1-release-notes.md    | 6 +++++-
 doc/release-notes/9464-json-validation.md | 3 ---
 2 files changed, 5 insertions(+), 4 deletions(-)
 delete mode 100644 doc/release-notes/9464-json-validation.md

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 06a3e01f7af..990ba219cad 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -24,7 +24,7 @@ With the upload-out-of-band option enabled, it is also possible for file upload
  Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
 
 Since Alternative Title is repeatable now, old json apis would not be compatable with a new version since value of alternative title has changed from simple string to an array.
-For example, instead "value": "Alternative Title", the value canbe "value": ["Alternative Title1", "Alternative Title2"]
+For example, instead "value": "Alternative Title", the value can be "value": ["Alternative Title1", "Alternative Title2"]
 
 ### Improvements in the /versions API
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
@@ -45,6 +45,8 @@ This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/
 - deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
 - getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
 - getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.
+- getDatasetJsonSchema (/api/dataverses/{id}/datasetSchema): Get a dataset schema with the fields required by a given dataverse collection.
+- validateDatasetJsonSchema (/api/dataverses/{id}/validateDatasetJson): Validate that a dataset json file is in proper format and contains the required elements and fields for a given dataverse collection.
 
 ### Extended the existing endpoints:
 - getVersionFiles (/api/datasets/{id}/versions/{versionId}/files): Extended to support optional filtering by search text through the `searchText` query parameter. The search will be applied to the labels and descriptions of the dataset files. Added `tabularTagName` to return files to which the particular tabular tag has been added. Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files.
@@ -112,6 +114,8 @@ to generate updated versions.
 - We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
 See also #10060.
 
+- Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release funtionality is limited to json format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
+
 ### Solr Improvements
 - As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
 
diff --git a/doc/release-notes/9464-json-validation.md b/doc/release-notes/9464-json-validation.md
deleted file mode 100644
index f104263ba35..00000000000
--- a/doc/release-notes/9464-json-validation.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release funtionality is limited to json format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
-
-For documentation see the API changelog: http://preview.guides.gdcc.io/en/develop/api/changelog.html

From 6d2f87ca93c108a9b4ec4905372a2e1709b3f5cf Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Wed, 6 Dec 2023 12:24:26 -0500
Subject: [PATCH 888/922] adding review comment changes

---
 doc/release-notes/6.1-release-notes.md | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 990ba219cad..4b5c20f3953 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -12,8 +12,8 @@ This release contains major upgrades to core components. Detailed upgrade instru
 ## Detailed Release Highlights, New Features and Use Case Scenarios
 
 ### Dataverse installation can be now be configured to allow out-of-band upload
-- Installation can be now be configured to allow out-of-band upload by setting the `dataverse.files.<id>.upload-out-of-band` JVM option to `true`.
-By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+In some situations, direct upload might not work from the UI, e.g., when s3 storage is not accessible from the internet. This pull request adds an option to [allow direct uploads via API only](https://github.com/IQSS/dataverse/pull/9003). This way, a third party application can use direct upload from within the internal network, while there is no direct download available to the users via UI.
+By default, Dataverse supports uploading files via the [add a file to a dataset](https://guides.dataverse.org/en/6.1/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
 With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
 ### Alternative Title is made repeatable.
@@ -23,7 +23,7 @@ With the upload-out-of-band option enabled, it is also possible for file upload
  Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml`
  Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
 
-Since Alternative Title is repeatable now, old json apis would not be compatable with a new version since value of alternative title has changed from simple string to an array.
+Since Alternative Title is repeatable now, old json apis would not be compatible with a new version since value of alternative title has changed from simple string to an array.
 For example, instead "value": "Alternative Title", the value can be "value": ["Alternative Title1", "Alternative Title2"]
 
 ### Improvements in the /versions API
@@ -70,7 +70,6 @@ This parameter applies a filter criteria to the operation and supports the follo
     - Can delete the dataset draft
 - getDatasetVersionCitation (/api/datasets/{id}/versions/{versionId}/citation) endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation.
 
-
 ### DataFile API payload has been extended to include the following fields:
 - tabularData: Boolean field to know if the DataFile is of tabular type
 - fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner)
@@ -114,7 +113,7 @@ to generate updated versions.
 - We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
 See also #10060.
 
-- Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release funtionality is limited to json format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
+- Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to json format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
 
 ### Solr Improvements
 - As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
@@ -125,12 +124,13 @@ Please see the "Installing Solr" section of the Installation Prerequisites guide
 ### Development
 - Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
 For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools
-
 - There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
-
 - A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
   - SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
   - Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
+- `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
+also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
+- As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
 
 ## OpenID Connect Authentication Provider Improvements
 
@@ -175,6 +175,8 @@ As part of these testing improvements, the code coverage report file for unit te
 - dataverse.auth.oidc.subtitle
 - dataverse.auth.oidc.pkce.max-cache-size
 - dataverse.auth.oidc.pkce.max-cache-age
+- dataverse.files.{driverId}.upload-out-of-band
+- dataverse.files.guestbook-at-request
 
 ## Installation
 
@@ -182,14 +184,17 @@ If this is a new installation, please follow our [Installation Guide](https://gu
 
 Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club!
 
-You are also very welcome to join the [Global Dataverse Community Consortium](https://dataversecommunity.global) (GDCC).
+You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC).
 
 ## Upgrade Instructions
-
 Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
 
 These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
 
+## Backward Incompatibilities
+- Since Alternative Title is repeatable now, old json apis would not be compatible with a new version
+- Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility,
+
 ## Complete List of Changes
 
 For the complete list of code changes in this release, see the [6.1 Milestone](https://github.com/IQSS/dataverse/milestone/110?closed=1) in GitHub.

From 90ff56ca979cd71f1c467ff1cfa0dfeb8f619691 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:43:43 -0500
Subject: [PATCH 889/922] Update doc/release-notes/6.1-release-notes.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/release-notes/6.1-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 4b5c20f3953..e1a9214a982 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -122,7 +122,7 @@ Please see the "Installing Solr" section of the Installation Prerequisites guide
 
 
 ### Development
-- Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
+- Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using Netbeans or IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
 For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools
 - There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
 - A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.

From 10e0e25fe10dda9f49b6126f591b9483adb2f765 Mon Sep 17 00:00:00 2001
From: Steven Winship <39765413+stevenwinship@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:44:49 -0500
Subject: [PATCH 890/922] Update doc/release-notes/6.1-release-notes.md

Co-authored-by: Philip Durbin <philip_durbin@harvard.edu>
---
 doc/release-notes/6.1-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index e1a9214a982..427a07a4c2c 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -123,7 +123,7 @@ Please see the "Installing Solr" section of the Installation Prerequisites guide
 
 ### Development
 - Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using Netbeans or IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
-For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools
+For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools and [the thread](https://groups.google.com/g/dataverse-community/c/zNBDzSMF2Q0/m/Z-xS6fA2BgAJ) on the mailing list.
 - There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
 - A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
   - SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.

From 3d55ed31de8fb9e45a2cedfecf07e22c82dae12a Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Wed, 6 Dec 2023 12:47:53 -0500
Subject: [PATCH 891/922] adding review comment changes

---
 doc/release-notes/6.1-release-notes.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 427a07a4c2c..189f21f2322 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -19,21 +19,22 @@ With the upload-out-of-band option enabled, it is also possible for file upload
 ### Alternative Title is made repeatable.
 - One will need to update database with updated citation block.
   `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
-- One will also need to update solr schema:
- Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml`
- Reload solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
+- One will also need to update Solr schema:
+ Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-9.3.0/server/solr/collection1/conf/schema.xml`
+ Reload Solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
 
-Since Alternative Title is repeatable now, old json apis would not be compatible with a new version since value of alternative title has changed from simple string to an array.
+Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version since value of alternative title has changed from simple string to an array.
 For example, instead "value": "Alternative Title", the value can be "value": ["Alternative Title1", "Alternative Title2"]
 
-### Improvements in the /versions API
+### Improvements in the dataset versions API
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
 - a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
 - when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files.
 
 This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide.
-### The following API endpoints have been added:
 
+### The following API endpoints have been added:
+- deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
 - /api/files/{id}/downloadCount
 - /api/files/{id}/dataTables 
 - /api/files/{id}/metadata/tabularTags New endpoint to set tabular file tags.
@@ -42,11 +43,10 @@ This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/
 - setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
 - userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file.
 - hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
-- deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
 - getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
 - getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.
 - getDatasetJsonSchema (/api/dataverses/{id}/datasetSchema): Get a dataset schema with the fields required by a given dataverse collection.
-- validateDatasetJsonSchema (/api/dataverses/{id}/validateDatasetJson): Validate that a dataset json file is in proper format and contains the required elements and fields for a given dataverse collection.
+- validateDatasetJsonSchema (/api/dataverses/{id}/validateDatasetJson): Validate that a dataset JSON file is in proper format and contains the required elements and fields for a given dataverse collection.
 
 ### Extended the existing endpoints:
 - getVersionFiles (/api/datasets/{id}/versions/{versionId}/files): Extended to support optional filtering by search text through the `searchText` query parameter. The search will be applied to the labels and descriptions of the dataset files. Added `tabularTagName` to return files to which the particular tabular tag has been added. Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files.
@@ -113,7 +113,7 @@ to generate updated versions.
 - We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
 See also #10060.
 
-- Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to json format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
+- Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
 
 ### Solr Improvements
 - As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
@@ -192,7 +192,7 @@ Upgrading requires a maintenance window and downtime. Please plan ahead, create
 These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
 
 ## Backward Incompatibilities
-- Since Alternative Title is repeatable now, old json apis would not be compatible with a new version
+- Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version
 - Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility,
 
 ## Complete List of Changes

From 1be5d4b6b2baddc5f30bf598d81bd5ed991f73ee Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Wed, 6 Dec 2023 12:52:39 -0500
Subject: [PATCH 892/922] adding review comment changes

---
 doc/release-notes/6.1-release-notes.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 189f21f2322..d0fe895565c 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -87,10 +87,8 @@ This parameter applies a filter criteria to the operation and supports the follo
 
 ### Misc
 - Configure tools are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
-
 - Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
 The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions.
-
 - Dataverse's OAI_ORE Metadata Export format and archival BagIT exports
 (which include the OAI-ORE metadata export file) have been updated to include
 information about the dataset version state, e.g. RELEASED or DEACCESSIONED
@@ -104,7 +102,7 @@ Dataverse installations that have been using archival Bags may wish to update an
 existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
 [archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
 to generate updated versions.
-
+- There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
 - This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
   - To fix #9952, we surround the license info with `<` and `>`.
   - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
@@ -124,7 +122,6 @@ Please see the "Installing Solr" section of the Installation Prerequisites guide
 ### Development
 - Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using Netbeans or IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
 For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools and [the thread](https://groups.google.com/g/dataverse-community/c/zNBDzSMF2Q0/m/Z-xS6fA2BgAJ) on the mailing list.
-- There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
 - A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
   - SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
   - Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.

From 8e2ff826bdd0f41e598a56012fa780d5f9148a2e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 13:41:35 -0500
Subject: [PATCH 893/922] store tests

---
 .../dataaccess/GlobusOverlayAccessIOTest.java | 148 ++++++++++++++++++
 1 file changed, 148 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
index e69de29bb2d..792a9974076 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
@@ -0,0 +1,148 @@
+/*
+ * SPDX-License-Identifier: Apache 2.0
+ */
+package edu.harvard.iq.dataverse.dataaccess;
+
+import edu.harvard.iq.dataverse.DOIServiceBean;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DvObject;
+import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.mocks.MocksFactory;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import static org.junit.jupiter.api.Assertions.*;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.STRICT_STUBS)
+public class GlobusOverlayAccessIOTest {
+
+    @Mock
+
+    private Dataset dataset;
+    private DataFile mDatafile;
+    private DataFile rDatafile;
+    private String baseStoreId1 = "182ad2bda2f-c3508e719076";
+    private String baseStoreId2 = "182ad2bda2f-c3508e719077";
+    private String logoPath = "d7c42580-6538-4605-9ad8-116a61982644/hdc1/image002.mrc";
+    private String authority = "10.5072";
+    private String identifier = "F2ABCDEF";
+
+    @BeforeEach
+    public void setUp() {
+        // Base Store
+        System.setProperty("dataverse.files.base.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+        System.setProperty("dataverse.files.base.label", "default");
+        System.setProperty("dataverse.files.base.directory", "/tmp/files");
+
+        // Managed Globus Store
+
+        // Nonsense endpoint/paths
+        System.setProperty("dataverse.files.globusm." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH,
+                "d7c42580-6538-4605-9ad8-116a61982644/hdc1");
+        // Nonsense value of the right form
+        System.setProperty("dataverse.files.globusm.globus-token",
+                "NzM2NTQxMDMtOTg1Yy00NDgzLWE1MTYtYTJlNDk0ZmI3MDhkOkpJZGZaZGxMZStQNUo3MTRIMDY2cDh6YzIrOXI2RmMrbFR6UG0zcSsycjA9");
+        System.setProperty("dataverse.files.globusm.remote-store-name", "GlobusEndpoint1");
+        System.setProperty("dataverse.files.globusm.type", "globus");
+        System.setProperty("dataverse.files.globusm.managed", "true");
+        System.setProperty("dataverse.files.globusm.base-store", "base");
+        System.setProperty("dataverse.files.globusm.label", "globusManaged");
+
+        // Remote Store
+        System.setProperty("dataverse.files.globusr.type", "globus");
+        System.setProperty("dataverse.files.globusr.base-store", "base");
+        System.setProperty("dataverse.files.globusr.managed", "false");
+        System.setProperty("dataverse.files.globusm.label", "globusRemote");
+        System.setProperty(
+                "dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS,
+                "d7c42580-6538-4605-9ad8-116a61982644/hdc1");
+        System.setProperty("dataverse.files.globusr.remote-store-name", "DemoDataCorp");
+        dataset = MocksFactory.makeDataset();
+        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, authority, identifier, "/",
+                DOIServiceBean.DOI_RESOLVER_URL, null));
+        mDatafile = MocksFactory.makeDataFile();
+        mDatafile.setOwner(dataset);
+        mDatafile.setStorageIdentifier("globusm://" + baseStoreId1);
+
+        rDatafile = MocksFactory.makeDataFile();
+        rDatafile.setOwner(dataset);
+        rDatafile.setStorageIdentifier("globusr://" + baseStoreId2 + "//" + logoPath);
+    }
+
+    @AfterEach
+    public void tearDown() {
+        System.clearProperty("dataverse.files.base.type");
+        System.clearProperty("dataverse.files.base.label");
+        System.clearProperty("dataverse.files.base.directory");
+        System.clearProperty("dataverse.files.globusm." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH);
+        System.clearProperty("dataverse.files.globusm.globus-token");
+        System.clearProperty("dataverse.files.globusm.remote-store-name");
+        System.clearProperty("dataverse.files.globusm.type");
+        System.clearProperty("dataverse.files.globusm.managed");
+        System.clearProperty("dataverse.files.globusm.base-store");
+        System.clearProperty("dataverse.files.globusm.label");
+        System.clearProperty("dataverse.files.globusr.type");
+        System.clearProperty("dataverse.files.globusr.base-store");
+        System.clearProperty("dataverse.files.globusr.managed");
+        System.clearProperty("dataverse.files.globusm.label");
+        System.clearProperty(
+                "dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS);
+        System.clearProperty("dataverse.files.globusr.remote-store-name");
+    }
+
+    @Test
+    void testGlobusOverlayIdentifiers() throws IOException {
+        assertTrue(GlobusOverlayAccessIO.isValidIdentifier("globusm", mDatafile.getStorageIdentifier()));
+        assertTrue(GlobusOverlayAccessIO.isValidIdentifier("globusr", rDatafile.getStorageIdentifier()));
+        assertFalse(GlobusOverlayAccessIO.isValidIdentifier("globusm", "globusr://localid//../of/the/hill"));
+        assertFalse(GlobusOverlayAccessIO.isValidIdentifier("globusr",
+                rDatafile.getStorageIdentifier().replace("hdc1", "")));
+
+        // We can read the storageIdentifier and get the driver
+        assertTrue(mDatafile.getStorageIdentifier()
+                .startsWith(DataAccess.getStorageDriverFromIdentifier(mDatafile.getStorageIdentifier())));
+        assertTrue(rDatafile.getStorageIdentifier()
+                .startsWith(DataAccess.getStorageDriverFromIdentifier(rDatafile.getStorageIdentifier())));
+
+        // We can get the driver type from it's ID
+        assertTrue(DataAccess.getDriverType("globusm").equals(System.getProperty("dataverse.files.globusm.type")));
+        assertTrue(DataAccess.getDriverType("globusr").equals(System.getProperty("dataverse.files.globusr.type")));
+
+        // When we get a StorageIO for the file, it is the right type
+        StorageIO<DataFile> mStorageIO = DataAccess.getStorageIO(mDatafile);
+        assertTrue(mStorageIO instanceof GlobusOverlayAccessIO);
+        StorageIO<DataFile> rStorageIO = DataAccess.getStorageIO(rDatafile);
+        assertTrue(rStorageIO instanceof GlobusOverlayAccessIO);
+
+        // When we use it, we can get properties like the remote store name
+        assertTrue(mStorageIO.getRemoteStoreName()
+                .equals(System.getProperty("dataverse.files.globusm.remote-store-name")));
+        assertTrue(rStorageIO.getRemoteStoreName()
+                .equals(System.getProperty("dataverse.files.globusr.remote-store-name")));
+
+        // Storage Locations are correct
+        String mLocation = mStorageIO.getStorageLocation();
+        assertEquals("globusm:///" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage()
+                + "/" + baseStoreId1, mLocation);
+        String rLocation = rStorageIO.getStorageLocation();
+        assertEquals("globusr://" + baseStoreId2 + "//" + logoPath, rLocation);
+
+        // If we ask for the path for an aux file, it is correct
+        System.out.println(Paths.get(System.getProperty("dataverse.files.file.directory", "/tmp/files"), authority,
+                identifier, baseStoreId1 + ".auxobject").toString());
+        System.out.println(mStorageIO.getAuxObjectAsPath("auxobject").toString());
+        assertTrue(Paths.get(System.getProperty("dataverse.files.base.directory", "/tmp/files"), authority, identifier,
+                baseStoreId1 + ".auxobject").equals(mStorageIO.getAuxObjectAsPath("auxobject")));
+        assertTrue(Paths.get(System.getProperty("dataverse.files.base.directory", "/tmp/files"), authority, identifier,
+                baseStoreId2 + ".auxobject").equals(rStorageIO.getAuxObjectAsPath("auxobject")));
+    }
+}

From 865c9feb4230a0a3bc9880cb6088a563b3fe21fc Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 13:53:35 -0500
Subject: [PATCH 894/922] getConfig tests

---
 .../iq/dataverse/dataaccess/StorageIOTest.java       | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java
index 2ed9d18036d..84a241b90f6 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java
@@ -243,4 +243,16 @@ public void testGenerateVariableHeader() {
         assertEquals("Random	Random\n", instance.generateVariableHeader(dvs));
         assertEquals(null, instance.generateVariableHeader(null));
     }
+    
+    @Test
+    public void testGetConfigParam() {
+        System.setProperty("dataverse.files.globus.type", "globus");
+    assertEquals("globus", StorageIO.getConfigParamForDriver("globus", StorageIO.TYPE));
+    System.clearProperty("dataverse.files.globus.type");
+    }
+    
+    @Test
+    public void testGetConfigParamWithDefault() {
+    assertEquals(DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER, StorageIO.getConfigParamForDriver("globus", AbstractRemoteOverlayAccessIO.BASE_STORE, DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER));
+    }
 }

From cb1beaae490126c2274219dfcb4cae56094b096a Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 6 Dec 2023 14:11:15 -0500
Subject: [PATCH 895/922] finish changing minio secret key #6783

This should have been part of 811d79a7
---
 docker-compose-dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index e68215d53d2..5265a6b7c2d 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -42,7 +42,7 @@ services:
         -Ddataverse.files.minio1.upload-redirect=false
         -Ddataverse.files.minio1.download-redirect=false
         -Ddataverse.files.minio1.access-key=4cc355_k3y
-        -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k35
+        -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y
     ports:
       - "8080:8080" # HTTP (Dataverse Application)
       - "4848:4848" # HTTP (Payara Admin Console)

From 5b7a560a380db12d083e82a19a865eb79559e0a4 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 14:41:31 -0500
Subject: [PATCH 896/922] refactor, test for getFileMap

---
 .../harvard/iq/dataverse/api/Datasets.java    |   3 +-
 .../AbstractRemoteOverlayAccessIO.java        |   2 +-
 .../dataverse/globus/GlobusServiceBean.java   | 134 +++++++++---------
 .../iq/dataverse/globus/GlobusUtil.java       |  33 +++++
 .../dataaccess/GlobusOverlayAccessIOTest.java |   1 -
 .../iq/dataverse/globus/GlobusUtilTest.java   |  88 ++++++++++++
 6 files changed, 190 insertions(+), 71 deletions(-)
 create mode 100644 src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java
 create mode 100644 src/test/java/edu/harvard/iq/dataverse/globus/GlobusUtilTest.java

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 939ebf1dcd4..b3bfc476423 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -110,6 +110,7 @@
 import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
 
 import edu.harvard.iq.dataverse.globus.GlobusServiceBean;
+import edu.harvard.iq.dataverse.globus.GlobusUtil;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -3996,7 +3997,7 @@ public Response requestGlobusDownload(@Context ContainerRequestContext crc, @Pat
             }
         }
         // Allowed to download all requested files
-        JsonObject files = globusService.getFilesMap(dataFiles, dataset);
+        JsonObject files = GlobusUtil.getFilesMap(dataFiles, dataset);
         if (GlobusAccessibleStore.isDataverseManaged(dataset.getEffectiveStorageDriverId())) {
             // If managed, give the principal read permissions
             int status = globusService.setPermissionForDownload(dataset, body.getString("principal"));
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
index 8d058b7c9e3..6c26502acfa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
@@ -42,7 +42,7 @@
 public abstract class AbstractRemoteOverlayAccessIO<T extends DvObject> extends StorageIO<T> {
 
     protected static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.RemoteOverlayAccessIO");
-    protected static final String REFERENCE_ENDPOINTS_WITH_BASEPATHS = "reference-endpoints-with-basepaths";
+    public static final String REFERENCE_ENDPOINTS_WITH_BASEPATHS = "reference-endpoints-with-basepaths";
     static final String BASE_STORE = "base-store";
     protected static final String SECRET_KEY = "secret-key";
     static final String URL_EXPIRATION_MINUTES = "url-expiration-minutes";
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 37959188857..8cc8e491416 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -159,9 +159,11 @@ public void deletePermission(String ruleId, Dataset dataset, Logger globusLogger
         }
     }
 
-    /** Request read/write access for the specified principal and generate a list of accessible paths for new files for the specified dataset.
+    /**
+     * Request read/write access for the specified principal and generate a list of
+     * accessible paths for new files for the specified dataset.
      * 
-     * @param principal - the id of the Globus principal doing the transfer
+     * @param principal     - the id of the Globus principal doing the transfer
      * @param dataset
      * @param numberOfPaths - how many files are to be transferred
      * @return
@@ -230,10 +232,15 @@ private int requestPermission(GlobusEndpoint endpoint, Dataset dataset, Permissi
         }
     }
 
-    /** Given an array of remote files to be referenced in the dataset, create a set of valid storage identifiers and return a map of the remote file paths to storage identifiers.
+    /**
+     * Given an array of remote files to be referenced in the dataset, create a set
+     * of valid storage identifiers and return a map of the remote file paths to
+     * storage identifiers.
      * 
      * @param dataset
-     * @param referencedFiles - a JSON array of remote files to be referenced in the dataset - each should be a string with the <Globus endpoint>/path/to/file
+     * @param referencedFiles - a JSON array of remote files to be referenced in the
+     *                        dataset - each should be a string with the <Globus
+     *                        endpoint>/path/to/file
      * @return - a map of supplied paths to valid storage identifiers
      */
     public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray referencedFiles) {
@@ -262,15 +269,17 @@ public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray ref
         return fileMap.build();
     }
 
-    
-    /** A cache of temporary permission requests - for upload (rw) and download (r) access.
-     * When a temporary permission request is created, it is added to the cache. After GLOBUS_CACHE_MAXAGE minutes, if a transfer has not been started, the permission will be revoked/deleted.
-     * (If a transfer has been started, the permission will not be revoked/deleted until the transfer is complete. This is handled in other methods.)
+    /**
+     * A cache of temporary permission requests - for upload (rw) and download (r)
+     * access. When a temporary permission request is created, it is added to the
+     * cache. After GLOBUS_CACHE_MAXAGE minutes, if a transfer has not been started,
+     * the permission will be revoked/deleted. (If a transfer has been started, the
+     * permission will not be revoked/deleted until the transfer is complete. This
+     * is handled in other methods.)
      */
     // Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
-            .expireAfterWrite(
-                    Duration.of(JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
+            .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
             .scheduler(Scheduler.systemScheduler()).evictionListener((ruleId, datasetId, cause) -> {
                 // Delete rules that expire
                 logger.fine("Rule " + ruleId + " expired");
@@ -280,20 +289,24 @@ public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray ref
 
             .build();
 
-    //Convenience method to add a temporary permission request to the cache - allows logging of temporary permission requests
+    // Convenience method to add a temporary permission request to the cache -
+    // allows logging of temporary permission requests
     private void monitorTemporaryPermissions(String ruleId, long datasetId) {
         logger.fine("Adding rule " + ruleId + " for dataset " + datasetId);
         rulesCache.put(ruleId, datasetId);
     }
 
-/** Call the Globus API to get info about the transfer.
- * 
- * @param accessToken
- * @param taskId - the Globus task id supplied by the user
- * @param globusLogger - the transaction-specific logger to use (separate log files are created in general, some calls may use the class logger)
- * @return
- * @throws MalformedURLException
- */
+    /**
+     * Call the Globus API to get info about the transfer.
+     * 
+     * @param accessToken
+     * @param taskId       - the Globus task id supplied by the user
+     * @param globusLogger - the transaction-specific logger to use (separate log
+     *                     files are created in general, some calls may use the
+     *                     class logger)
+     * @return
+     * @throws MalformedURLException
+     */
     public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger) throws MalformedURLException {
 
         URL url = new URL("https://transfer.api.globusonline.org/v0.10/endpoint_manager/task/" + taskId);
@@ -313,9 +326,12 @@ public GlobusTask getTask(String accessToken, String taskId, Logger globusLogger
         return task;
     }
 
-    /** Globus call to get an access token for the user using the long-term token we hold.
+    /**
+     * Globus call to get an access token for the user using the long-term token we
+     * hold.
      * 
-     * @param globusBasicToken - the base64 encoded Globus Basic token comprised of the <Globus user id>:<key>
+     * @param globusBasicToken - the base64 encoded Globus Basic token comprised of
+     *                         the <Globus user id>:<key>
      * @return - a valid Globus access token
      */
     public static AccessToken getClientToken(String globusBasicToken) {
@@ -433,7 +449,6 @@ static class MakeRequestResponse {
 
     }
 
-
     /**
      * Cache of open download Requests This cache keeps track of the set of files
      * selected for transfer out (download) via Globus. It is a means of
@@ -480,10 +495,11 @@ public String getGlobusAppUrlForDataset(Dataset d) {
         return getGlobusAppUrlForDataset(d, true, null);
     }
 
-    /** Generated the App URl for upload (in) or download (out)
+    /**
+     * Generated the App URl for upload (in) or download (out)
      * 
-     * @param d - the dataset involved
-     * @param upload - boolean, true for upload, false for download
+     * @param d         - the dataset involved
+     * @param upload    - boolean, true for upload, false for download
      * @param dataFiles - a list of the DataFiles to be downloaded
      * @return
      */
@@ -516,7 +532,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<DataFile
                     + "/globusUploadParameters?locale=" + localeCode;
         } else {
             // Download
-            JsonObject files = getFilesMap(dataFiles, d);
+            JsonObject files = GlobusUtil.getFilesMap(dataFiles, d);
 
             String downloadId = UUID.randomUUID().toString();
             downloadCache.put(downloadId, files);
@@ -540,27 +556,8 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, List<DataFile
         return finalUrl;
     }
 
-    public JsonObject getFilesMap(List<DataFile> dataFiles, Dataset d) {
-        JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
-        for (DataFile df : dataFiles) {
-            String storageId = df.getStorageIdentifier();
-            String[] parts = DataAccess.getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(storageId, d));
-            String driverId =  parts[0];
-            String fileLocation = parts[1];
-            if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
-                String endpointWithBasePath = GlobusAccessibleStore.getTransferEnpointWithPath(driverId);
-                fileLocation = endpointWithBasePath + "/" + fileLocation;
-            } else {
-                fileLocation = storageId.substring(storageId.lastIndexOf("//") + 2);
-            }
-            filesBuilder.add(df.getId().toString(), fileLocation);
-        }
-        return filesBuilder.build();
-    }
-
     private String getGlobusDownloadScript(Dataset dataset, ApiToken apiToken, List<DataFile> downloadDFList) {
         return URLTokenUtil.getScriptForUrl(getGlobusAppUrlForDataset(dataset, false, downloadDFList));
-
     }
 
     @Asynchronous
@@ -608,8 +605,8 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
                 rulesCache.invalidate(ruleId);
             }
         }
-        
-        //Wait before first check
+
+        // Wait before first check
         Thread.sleep(5000);
         // globus task status check
         task = globusStatusCheck(endpoint, taskIdentifier, globusLogger);
@@ -907,8 +904,8 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro
         }
         task = globusStatusCheck(endpoint, taskIdentifier, globusLogger);
         String taskStatus = getTaskStatus(task);
-        
-        //Transfer is done (success or failure) so delete the rule
+
+        // Transfer is done (success or failure) so delete the rule
         if (ruleId != null) {
             logger.info("Deleting: rule: " + ruleId);
             deletePermission(ruleId, dataset, globusLogger);
@@ -1150,13 +1147,14 @@ private GlobusEndpoint getGlobusEndpoint(DvObject dvObject) {
 
         return endpoint;
     }
-    
+
     // This helper method is called from the Download terms/guestbook/etc. popup,
     // when the user clicks the "ok" button. We use it, instead of calling
     // downloadServiceBean directly, in order to differentiate between single
     // file downloads and multiple (batch) downloads - since both use the same
     // terms/etc. popup.
-    public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, boolean doNotSaveGuestbookResponse) {
+    public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse,
+            boolean doNotSaveGuestbookResponse) {
         PrimeFaces.current().executeScript("PF('guestbookAndTermsPopup').hide()");
         guestbookResponse.setEventType(GuestbookResponse.DOWNLOAD);
 
@@ -1170,7 +1168,7 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse,
             apiToken = new ApiToken();
             apiToken.setTokenString(privUrl.getToken());
         }
-        
+
         DataFile df = guestbookResponse.getDataFile();
         if (df != null) {
             logger.fine("Single datafile case for writeGuestbookAndStartTransfer");
@@ -1179,35 +1177,35 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse,
             if (!doNotSaveGuestbookResponse) {
                 fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
             }
-            PrimeFaces.current()
-                    .executeScript(getGlobusDownloadScript(df.getOwner(), apiToken, downloadDFList));
+            PrimeFaces.current().executeScript(getGlobusDownloadScript(df.getOwner(), apiToken, downloadDFList));
         } else {
-            //Following FileDownloadServiceBean writeGuestbookAndStartBatchDownload
+            // Following FileDownloadServiceBean writeGuestbookAndStartBatchDownload
             List<String> list = new ArrayList<>(Arrays.asList(guestbookResponse.getSelectedFileIds().split(",")));
             List<DataFile> selectedFiles = new ArrayList<DataFile>();
             for (String idAsString : list) {
                 try {
                     Long fileId = Long.parseLong(idAsString);
-                // If we need to create a GuestBookResponse record, we have to 
-                // look up the DataFile object for this file: 
-                if (!doNotSaveGuestbookResponse) {
-                    df = dataFileService.findCheapAndEasy(fileId);
-                    guestbookResponse.setDataFile(df);
-                    fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
-                    selectedFiles.add(df);
-                }
+                    // If we need to create a GuestBookResponse record, we have to
+                    // look up the DataFile object for this file:
+                    if (!doNotSaveGuestbookResponse) {
+                        df = dataFileService.findCheapAndEasy(fileId);
+                        guestbookResponse.setDataFile(df);
+                        fileDownloadService.writeGuestbookResponseRecord(guestbookResponse);
+                        selectedFiles.add(df);
+                    }
                 } catch (NumberFormatException nfe) {
-                    logger.warning("A file id passed to the writeGuestbookAndStartTransfer method as a string could not be converted back to Long: " + idAsString);
+                    logger.warning(
+                            "A file id passed to the writeGuestbookAndStartTransfer method as a string could not be converted back to Long: "
+                                    + idAsString);
                     return;
                 }
 
             }
             if (!selectedFiles.isEmpty()) {
-                //Use dataset from one file - files should all be from the same dataset
-                PrimeFaces.current().executeScript(getGlobusDownloadScript(df.getOwner(), apiToken,
-                        selectedFiles));
+                // Use dataset from one file - files should all be from the same dataset
+                PrimeFaces.current().executeScript(getGlobusDownloadScript(df.getOwner(), apiToken, selectedFiles));
             }
         }
-     }
+    }
 
 }
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java
new file mode 100644
index 00000000000..92cf8ac7704
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusUtil.java
@@ -0,0 +1,33 @@
+package edu.harvard.iq.dataverse.globus;
+
+import java.util.List;
+
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
+import jakarta.json.Json;
+import jakarta.json.JsonObject;
+import jakarta.json.JsonObjectBuilder;
+
+public class GlobusUtil {
+
+    public static JsonObject getFilesMap(List<DataFile> dataFiles, Dataset d) {
+        JsonObjectBuilder filesBuilder = Json.createObjectBuilder();
+        for (DataFile df : dataFiles) {
+            String storageId = df.getStorageIdentifier();
+            String[] parts = DataAccess
+                    .getDriverIdAndStorageLocation(DataAccess.getLocationFromStorageId(storageId, d));
+            String driverId = parts[0];
+            String fileLocation = parts[1];
+            if (GlobusAccessibleStore.isDataverseManaged(driverId)) {
+                String endpointWithBasePath = GlobusAccessibleStore.getTransferEnpointWithPath(driverId);
+                fileLocation = endpointWithBasePath + "/" + fileLocation;
+            } else {
+                fileLocation = storageId.substring(storageId.lastIndexOf("//") + 2);
+            }
+            filesBuilder.add(df.getId().toString(), fileLocation);
+        }
+        return filesBuilder.build();
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
index 792a9974076..856d71d7dc0 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
@@ -6,7 +6,6 @@
 import edu.harvard.iq.dataverse.DOIServiceBean;
 import edu.harvard.iq.dataverse.DataFile;
 import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DvObject;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.mocks.MocksFactory;
 import org.junit.jupiter.api.AfterEach;
diff --git a/src/test/java/edu/harvard/iq/dataverse/globus/GlobusUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/globus/GlobusUtilTest.java
new file mode 100644
index 00000000000..56f8731b9c8
--- /dev/null
+++ b/src/test/java/edu/harvard/iq/dataverse/globus/GlobusUtilTest.java
@@ -0,0 +1,88 @@
+package edu.harvard.iq.dataverse.globus;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+
+import edu.harvard.iq.dataverse.DOIServiceBean;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.dataaccess.AbstractRemoteOverlayAccessIO;
+import edu.harvard.iq.dataverse.dataaccess.DataAccess;
+import edu.harvard.iq.dataverse.dataaccess.GlobusAccessibleStore;
+import edu.harvard.iq.dataverse.mocks.MocksFactory;
+import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import jakarta.json.JsonObject;
+
+public class GlobusUtilTest {
+
+    private Dataset dataset;
+    private DataFile mDatafile;
+    private DataFile rDatafile;
+    private String baseStoreId1 = "182ad2bda2f-c3508e719076";
+    private String baseStoreId2 = "182ad2bda2f-c3508e719077";
+    private String logoPath = "d7c42580-6538-4605-9ad8-116a61982644/hdc1/image002.mrc";
+    private String authority = "10.5072";
+    private String identifier = "F2ABCDEF";
+
+    @BeforeEach
+    public void setUp() {
+
+        // Managed Globus Store
+
+        // Nonsense endpoint/paths
+        System.setProperty("dataverse.files.globusm." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH,
+                "d7c42580-6538-4605-9ad8-116a61982644/hdc1");
+        System.setProperty("dataverse.files.globusm.managed", "true");
+
+        // Remote Store
+        System.setProperty("dataverse.files.globusr.managed", "false");
+        System.setProperty(
+                "dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS,
+                "d7c42580-6538-4605-9ad8-116a61982644/hdc1");
+
+        dataset = MocksFactory.makeDataset();
+        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, authority, identifier, "/",
+                DOIServiceBean.DOI_RESOLVER_URL, null));
+        mDatafile = MocksFactory.makeDataFile();
+        mDatafile.setOwner(dataset);
+        mDatafile.setStorageIdentifier("globusm://" + baseStoreId1);
+
+        rDatafile = MocksFactory.makeDataFile();
+        rDatafile.setOwner(dataset);
+        rDatafile.setStorageIdentifier("globusr://" + baseStoreId2 + "//" + logoPath);
+        List<DataFile> files = new ArrayList<DataFile>();
+        files.add(mDatafile);
+        files.add(rDatafile);
+        dataset.setFiles(files);
+    }
+
+    @AfterEach
+    public void tearDown() {
+        System.clearProperty("dataverse.files.globusm." + GlobusAccessibleStore.TRANSFER_ENDPOINT_WITH_BASEPATH);
+        System.clearProperty("dataverse.files.globusm.managed");
+        System.clearProperty("dataverse.files.globusr.managed");
+        System.clearProperty(
+                "dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS);
+    }
+
+    
+    @Test
+    public void testgetFilesMap() {
+        
+        JsonObject jo = GlobusUtil.getFilesMap(dataset.getFiles(), dataset);
+        System.out.println(JsonUtil.prettyPrint(jo));
+        assertEquals(jo.getString(Long.toString(mDatafile.getId())), "d7c42580-6538-4605-9ad8-116a61982644/hdc1/10.5072/F2ABCDEF/182ad2bda2f-c3508e719076");
+        assertEquals(jo.getString(Long.toString(rDatafile.getId())), logoPath);
+    }
+}

From 4ba629d643678acdd0b649128b8a76a805ee6906 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Wed, 6 Dec 2023 15:28:32 -0500
Subject: [PATCH 897/922] adding review comment changes

---
 doc/release-notes/6.1-release-notes.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index d0fe895565c..38b99e6580b 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -11,6 +11,10 @@ This release contains major upgrades to core components. Detailed upgrade instru
 
 ## Detailed Release Highlights, New Features and Use Case Scenarios
 
+### Optional support for guestbooks to appear when files access is requested rather than after access has been granted and a download is started
+Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
+  The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions.
+
 ### Dataverse installation can be now be configured to allow out-of-band upload
 In some situations, direct upload might not work from the UI, e.g., when s3 storage is not accessible from the internet. This pull request adds an option to [allow direct uploads via API only](https://github.com/IQSS/dataverse/pull/9003). This way, a third party application can use direct upload from within the internal network, while there is no direct download available to the users via UI.
 By default, Dataverse supports uploading files via the [add a file to a dataset](https://guides.dataverse.org/en/6.1/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
@@ -142,7 +146,7 @@ life easier during instance setups and reconfiguration. You no longer need to ge
 necessary JSON file.
 
 ### Adding PKCE Support
-
+[This PR adds PKCE support for OIDC providers](https://github.com/IQSS/dataverse/pull/9273)
 Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
 support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
 

From 93d9b35a07625622523a4490eee8f55d617defec Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 16:32:17 -0500
Subject: [PATCH 898/922] future test code - requires config of Globus stores

---
 .../harvard/iq/dataverse/api/DatasetsIT.java  | 53 +++++++++++++++++++
 .../edu/harvard/iq/dataverse/api/UtilIT.java  | 32 +++++++++++
 .../dataaccess/GlobusOverlayAccessIOTest.java | 34 ++++++------
 3 files changed, 104 insertions(+), 15 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
index 6a746b7c5b5..928574eb82b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java
@@ -42,6 +42,9 @@
 
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
+import edu.harvard.iq.dataverse.dataaccess.AbstractRemoteOverlayAccessIO;
+import edu.harvard.iq.dataverse.dataaccess.GlobusOverlayAccessIOTest;
+import edu.harvard.iq.dataverse.dataaccess.StorageIO;
 import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
 
 import org.apache.commons.lang3.StringUtils;
@@ -135,6 +138,7 @@ public static void setUpClass() {
                 .statusCode(200);
          */
     }
+    
 
     @AfterAll
     public static void afterClass() {
@@ -4175,4 +4179,53 @@ public void testGetUserPermissionsOnDataset() {
         Response getUserPermissionsOnDatasetInvalidIdResponse = UtilIT.getUserPermissionsOnDataset("testInvalidId", apiToken);
         getUserPermissionsOnDatasetInvalidIdResponse.then().assertThat().statusCode(BAD_REQUEST.getStatusCode());
     }
+    
+    //Requires that a Globus remote store be set up as with the parameters in the GlobusOverlayAccessIOTest class
+    //Tests whether the API call succeeds and has some of the expected parameters
+    @Test
+    @Disabled
+    public void testGetGlobusUploadParameters() {
+        //Creates managed and remote Globus stores
+        GlobusOverlayAccessIOTest.setUp();
+
+        Response createUser = UtilIT.createRandomUser();
+        createUser.then().assertThat().statusCode(OK.getStatusCode());
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+        String username = UtilIT.getUsernameFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id");
+        
+        Response makeSuperUser = UtilIT.makeSuperUser(username);
+        assertEquals(200, makeSuperUser.getStatusCode());
+        
+        Response setDriver = UtilIT.setDatasetStorageDriver(datasetId, System.getProperty("dataverse.files.globusr.label"), apiToken);
+        assertEquals(200, setDriver.getStatusCode());
+        
+        Response getUploadParams = UtilIT.getDatasetGlobusUploadParameters(datasetId, "en_us", apiToken);
+        assertEquals(200, getUploadParams.getStatusCode());
+        JsonObject data = JsonUtil.getJsonObject(getUploadParams.getBody().asString());
+        JsonObject queryParams = data.getJsonObject("queryParameters");
+        assertEquals("en_us", queryParams.getString("dvLocale"));
+        assertEquals("false", queryParams.getString("managed"));
+        //Assumes only one reference endpoint with a basepath is configured
+        assertTrue(queryParams.getJsonArray("referenceEndpointsWithPaths").get(0).toString().indexOf(System.getProperty("dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS)) > -1);
+        JsonArray signedUrls = data.getJsonArray("signedUrls");
+        boolean found = false;
+        for (int i = 0; i < signedUrls.size(); i++) {
+            JsonObject signedUrl = signedUrls.getJsonObject(i);
+            if (signedUrl.getString("name").equals("requestGlobusReferencePaths")) {
+                found=true;
+                break;
+            }
+        }
+        assertTrue(found);
+        //Removes managed and remote Globus stores
+        GlobusOverlayAccessIOTest.tearDown();
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
index 869e755a183..bd2fe7e6f0b 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java
@@ -3636,4 +3636,36 @@ static Response downloadTmpFile(String fullyQualifiedPathToFile, String apiToken
                 .get("/api/admin/downloadTmpFile?fullyQualifiedPathToFile=" + fullyQualifiedPathToFile);
     }
 
+    static Response setDatasetStorageDriver(Integer datasetId, String driverLabel, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(driverLabel)
+                .put("/api/datasets/" + datasetId + "/storageDriver");
+    }
+    
+    
+    //Globus Store related - not currently used
+    
+    static Response getDatasetGlobusUploadParameters(Integer datasetId, String locale, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .contentType("application/json")
+                .get("/api/datasets/" + datasetId + "/globusUploadParameters?locale=" + locale);
+    }
+    
+    static Response getDatasetGlobusDownloadParameters(Integer datasetId, String locale, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .contentType("application/json")
+                .get("/api/datasets/" + datasetId + "/globusDownloadParameters?locale=" + locale);
+    }
+    
+    static Response requestGlobusDownload(Integer datasetId, JsonObject body, String apiToken) {
+        return given()
+                .header(API_TOKEN_HTTP_HEADER, apiToken)
+                .body(body)
+                .contentType("application/json")
+                .post("/api/datasets/" + datasetId + "/requestGlobusDownload");
+    }
+
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
index 856d71d7dc0..1c84fa90a9e 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
@@ -8,8 +8,9 @@
 import edu.harvard.iq.dataverse.Dataset;
 import edu.harvard.iq.dataverse.GlobalId;
 import edu.harvard.iq.dataverse.mocks.MocksFactory;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import static org.junit.jupiter.api.Assertions.*;
@@ -35,8 +36,8 @@ public class GlobusOverlayAccessIOTest {
     private String authority = "10.5072";
     private String identifier = "F2ABCDEF";
 
-    @BeforeEach
-    public void setUp() {
+    @BeforeAll
+    public static void setUp() {
         // Base Store
         System.setProperty("dataverse.files.base.type", DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
         System.setProperty("dataverse.files.base.label", "default");
@@ -65,20 +66,11 @@ public void setUp() {
                 "dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS,
                 "d7c42580-6538-4605-9ad8-116a61982644/hdc1");
         System.setProperty("dataverse.files.globusr.remote-store-name", "DemoDataCorp");
-        dataset = MocksFactory.makeDataset();
-        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, authority, identifier, "/",
-                DOIServiceBean.DOI_RESOLVER_URL, null));
-        mDatafile = MocksFactory.makeDataFile();
-        mDatafile.setOwner(dataset);
-        mDatafile.setStorageIdentifier("globusm://" + baseStoreId1);
 
-        rDatafile = MocksFactory.makeDataFile();
-        rDatafile.setOwner(dataset);
-        rDatafile.setStorageIdentifier("globusr://" + baseStoreId2 + "//" + logoPath);
     }
 
-    @AfterEach
-    public void tearDown() {
+    @AfterAll
+    public static void tearDown() {
         System.clearProperty("dataverse.files.base.type");
         System.clearProperty("dataverse.files.base.label");
         System.clearProperty("dataverse.files.base.directory");
@@ -100,6 +92,18 @@ public void tearDown() {
 
     @Test
     void testGlobusOverlayIdentifiers() throws IOException {
+
+        dataset = MocksFactory.makeDataset();
+        dataset.setGlobalId(new GlobalId(DOIServiceBean.DOI_PROTOCOL, authority, identifier, "/",
+                DOIServiceBean.DOI_RESOLVER_URL, null));
+        mDatafile = MocksFactory.makeDataFile();
+        mDatafile.setOwner(dataset);
+        mDatafile.setStorageIdentifier("globusm://" + baseStoreId1);
+
+        rDatafile = MocksFactory.makeDataFile();
+        rDatafile.setOwner(dataset);
+        rDatafile.setStorageIdentifier("globusr://" + baseStoreId2 + "//" + logoPath);
+
         assertTrue(GlobusOverlayAccessIO.isValidIdentifier("globusm", mDatafile.getStorageIdentifier()));
         assertTrue(GlobusOverlayAccessIO.isValidIdentifier("globusr", rDatafile.getStorageIdentifier()));
         assertFalse(GlobusOverlayAccessIO.isValidIdentifier("globusm", "globusr://localid//../of/the/hill"));

From 12b7c306dd31ebd987a2bae5f36dae27e4f0ba56 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 16:32:24 -0500
Subject: [PATCH 899/922] typo

---
 .../iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
index 1c84fa90a9e..ad980aa28cd 100644
--- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIOTest.java
@@ -61,7 +61,7 @@ public static void setUp() {
         System.setProperty("dataverse.files.globusr.type", "globus");
         System.setProperty("dataverse.files.globusr.base-store", "base");
         System.setProperty("dataverse.files.globusr.managed", "false");
-        System.setProperty("dataverse.files.globusm.label", "globusRemote");
+        System.setProperty("dataverse.files.globusr.label", "globusRemote");
         System.setProperty(
                 "dataverse.files.globusr." + AbstractRemoteOverlayAccessIO.REFERENCE_ENDPOINTS_WITH_BASEPATHS,
                 "d7c42580-6538-4605-9ad8-116a61982644/hdc1");

From 1426dfb6fc52ace869e3c822a732d5b408ca7c4c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Wed, 6 Dec 2023 16:47:54 -0500
Subject: [PATCH 900/922] add missing setting to release notes, add a todo to
 use two delays

---
 doc/release-notes/10162-globus-support.md                  | 7 ++++++-
 .../edu/harvard/iq/dataverse/globus/GlobusServiceBean.java | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/10162-globus-support.md b/doc/release-notes/10162-globus-support.md
index d64e72b70a1..7bc3990f840 100644
--- a/doc/release-notes/10162-globus-support.md
+++ b/doc/release-notes/10162-globus-support.md
@@ -1,4 +1,6 @@
-Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible, and for referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/latest/developers/big-data-support.html#big-data-support)
+Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible, 
+and for referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. 
+Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/latest/developers/big-data-support.html#big-data-support)
 - Globus functionality remains 'experimental'/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incomatibilities.
 - The functionality is configured per store and replaces the previous single-S3-Connector-per-Dataverse-instance model
 - Adding files to a dataset, and accessing files is supported via the Dataverse user interface through a separate [dataverse-globus app](https://github.com/scholarsportal/dataverse-globus)
@@ -10,5 +12,8 @@ Backward Incompatibilities:
 
 New JVM Options:
 - A new 'globus' store type and associated store-related options have been added. These are described in the [File Storage Options section of the Dataverse Guides](https://guides.dataverse.org/en/latest/installation/config.html#file-storage-using-a-local-filesystem-and-or-swift-and-or-object-stores-and-or-trusted-remote-stores).
+- dataverse.files.globus-cache-maxage - specifies the number of minutes Dataverse will wait between an initial request for a file transfer occurs and when that transfer must begin.
+
+
 
 Obsolete Settings: the :GlobusBasicToken, :GlobusEndpoint, and :GlobusStores settings are no longer used
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index 8cc8e491416..d0660a55a6a 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -277,6 +277,8 @@ public JsonObject requestReferenceFileIdentifiers(Dataset dataset, JsonArray ref
      * permission will not be revoked/deleted until the transfer is complete. This
      * is handled in other methods.)
      */
+    // ToDo - nominally this doesn't need to be as long as the allowed time for the
+    // downloadCache so there could be two separate settings.
     // Single cache of open rules/permission requests
     private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
             .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))

From d2427bd39046f104c95e27d1869d1665b969724f Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 7 Dec 2023 09:49:52 -0500
Subject: [PATCH 901/922] #10151 incorporate recent additions

---
 doc/release-notes/6.1-release-notes.md      | 22 +++++++++++++++++++++
 doc/release-notes/8549-collection-quotas.md |  3 ---
 doc/release-notes/8760-bagit.md             | 15 --------------
 3 files changed, 22 insertions(+), 18 deletions(-)
 delete mode 100644 doc/release-notes/8549-collection-quotas.md
 delete mode 100644 doc/release-notes/8760-bagit.md

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 38b99e6580b..38a7a1064e6 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -30,6 +30,28 @@ With the upload-out-of-band option enabled, it is also possible for file upload
 Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version since value of alternative title has changed from simple string to an array.
 For example, instead "value": "Alternative Title", the value can be "value": ["Alternative Title1", "Alternative Title2"]
 
+### Collection Storage Size Quota Support
+-This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
+Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of 6.1
+
+### BagIT Export Configurations Updated
+For BagIT export, it is now possible to configure the following information in bag-info.txt...
+
+Source-Organization: Harvard Dataverse
+Organization-Address: 1737 Cambridge Street, Cambridge, MA, USA
+Organization-Email: support@dataverse.harvard.edu
+
+... using new JVM/MPCONFIG options:
+
+- dataverse.bagit.sourceorg.name
+- dataverse.bagit.sourceorg.address
+- dataverse.bagit.sourceorg.email
+
+Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.
+
+For details, see https://dataverse-guide--10122.org.readthedocs.build/en/10122/installation/config.html#bag-info-txt
+
+
 ### Improvements in the dataset versions API
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
 - a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
diff --git a/doc/release-notes/8549-collection-quotas.md b/doc/release-notes/8549-collection-quotas.md
deleted file mode 100644
index b3635d0c5a1..00000000000
--- a/doc/release-notes/8549-collection-quotas.md
+++ /dev/null
@@ -1,3 +0,0 @@
-This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
-Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of 6.1
-
diff --git a/doc/release-notes/8760-bagit.md b/doc/release-notes/8760-bagit.md
deleted file mode 100644
index 30601857309..00000000000
--- a/doc/release-notes/8760-bagit.md
+++ /dev/null
@@ -1,15 +0,0 @@
-For BagIT export, it is now possible to configure the following information in bag-info.txt...
-
-Source-Organization: Harvard Dataverse
-Organization-Address: 1737 Cambridge Street, Cambridge, MA, USA
-Organization-Email: support@dataverse.harvard.edu
-
-... using new JVM/MPCONFIG options:
-
-- dataverse.bagit.sourceorg.name
-- dataverse.bagit.sourceorg.address
-- dataverse.bagit.sourceorg.email
-
-Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.
-
-For details, see https://dataverse-guide--10122.org.readthedocs.build/en/10122/installation/config.html#bag-info-txt

From 05c53066ea26c809b6376051ff336f11a4bcee9d Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 7 Dec 2023 10:29:47 -0500
Subject: [PATCH 902/922] mention download tmp file API #10151

---
 doc/release-notes/6.1-release-notes.md      | 1 +
 doc/release-notes/8760-download-tmp-file.md | 3 ---
 2 files changed, 1 insertion(+), 3 deletions(-)
 delete mode 100644 doc/release-notes/8760-download-tmp-file.md

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 38a7a1064e6..1b4e884cded 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -73,6 +73,7 @@ This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/
 - getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.
 - getDatasetJsonSchema (/api/dataverses/{id}/datasetSchema): Get a dataset schema with the fields required by a given dataverse collection.
 - validateDatasetJsonSchema (/api/dataverses/{id}/validateDatasetJson): Validate that a dataset JSON file is in proper format and contains the required elements and fields for a given dataverse collection.
+- downloadTmpFile (/api/admin/downloadTmpFile): For testing purposes, allows files to be downloaded from /tmp.
 
 ### Extended the existing endpoints:
 - getVersionFiles (/api/datasets/{id}/versions/{versionId}/files): Extended to support optional filtering by search text through the `searchText` query parameter. The search will be applied to the labels and descriptions of the dataset files. Added `tabularTagName` to return files to which the particular tabular tag has been added. Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files.
diff --git a/doc/release-notes/8760-download-tmp-file.md b/doc/release-notes/8760-download-tmp-file.md
deleted file mode 100644
index 7623a91ac9a..00000000000
--- a/doc/release-notes/8760-download-tmp-file.md
+++ /dev/null
@@ -1,3 +0,0 @@
-A new API has been added for testing purposes that allows files to be downloaded from /tmp.
-
-See

From 97c33218fa7224c544657e72f52c27d9cd8951bf Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 7 Dec 2023 10:30:23 -0500
Subject: [PATCH 903/922] remove duplicate "new" heading in API changelog
 #10151

---
 doc/sphinx-guides/source/api/changelog.rst | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index d2908533a14..910134e14f3 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -12,9 +12,6 @@ New
 ~~~
 - **/api/dataverses/{id}/datasetSchema**: See :ref:`get-dataset-json-schema`.
 - **/api/dataverses/{id}/validateDatasetJson**: See :ref:`validate-dataset-json`.
-
-New
-~~~
 - **/api/admin/clearThumbnailFailureFlag**: See :ref:`thumbnail_reset`.
 - **/api/admin/downloadTmpFile**: See :ref:`download-file-from-tmp`.
 

From 3a13ac8c56385ed2cc82bcc9db4f57fea7688a67 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 7 Dec 2023 10:34:21 -0500
Subject: [PATCH 904/922] #10151 add upgrade instructions

---
 doc/release-notes/6.1-release-notes.md        | 81 +++++++++++++++++++
 .../9002_allow_direct_upload_setting.md       |  5 --
 2 files changed, 81 insertions(+), 5 deletions(-)
 delete mode 100644 doc/release-notes/9002_allow_direct_upload_setting.md

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 38a7a1064e6..d5972338124 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -51,6 +51,13 @@ Previously, customization was possible by editing `Bundle.properties` but this i
 
 For details, see https://dataverse-guide--10122.org.readthedocs.build/en/10122/installation/config.html#bag-info-txt
 
+### Direct Upload setting added
+A Dataverse installation can be now be configured to allow out-of-band upload by setting the `dataverse.files.<id>.upload-out-of-band` JVM option to `true`.
+
+By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+
 
 ### Improvements in the dataset versions API
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
@@ -138,6 +145,7 @@ to generate updated versions.
 See also #10060.
 
 - Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
+- Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. (issue 9547). This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed.
 
 ### Solr Improvements
 - As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
@@ -214,6 +222,79 @@ Upgrading requires a maintenance window and downtime. Please plan ahead, create
 
 These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
 
+0\. These instructions assume that you are upgrading from 6.0. If you are running an earlier version, the only safe way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to 5.14. 
+
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
+
+In the following commands we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed.
+
+`export PAYARA=/usr/local/payara6`
+
+(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell)
+
+1\. Undeploy the previous version.
+
+- `$PAYARA/bin/asadmin undeploy dataverse-6.0`
+
+2\. Stop Payara and remove the generated directory
+
+- `service payara stop`
+- `rm -rf $PAYARA/glassfish/domains/domain1/generated`
+
+3\. Start Payara
+
+- `service payara start`
+
+4\. Deploy this version.
+
+- `$PAYARA/bin/asadmin deploy dataverse-6.1.war`
+
+5\. Restart Payara
+
+- `service payara stop`
+- `service payara start`
+
+6\. Update Geospatial Metadata Block (to improve validation of bounding box values)
+
+- `wget https://github.com/IQSS/dataverse/releases/download/v6.1/geospatial.tsv`
+- `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file @geospatial.tsv`
+
+6a\. Update Citation Metadata Block (to make Alternative Title repeatable)
+
+- `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
+
+7\. Upate Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with  custom metadata blocks  (7b).
+
+7a\. For installations without custom or experimental metadata blocks:
+
+- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/5.14/installation/prerequisites.html#solr-init-script))
+
+- Replace schema.xml
+
+  - `cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf`
+
+- Start Solr instance (usually `service solr start`, depending on Solr/OS)
+
+7b\. For installations with custom or experimental metadata blocks:
+
+- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/5.14/installation/prerequisites.html#solr-init-script))
+
+- There are 2 ways to regenerate the schema: Either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed):
+```
+	wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.3.0/update-fields.sh
+	chmod +x update-fields.sh
+	curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.3.0/server/solr/collection1/conf/schema.xml
+```
+OR, alternatively, you can edit the following line in your schema.xml by hand as follows (to indicate that alternative title is now `multiValued="true"`):
+```
+     <field name="alternativeTitle" type="text_en" multiValued="true" stored="true" indexed="true"/>
+```
+     
+- Restart Solr instance (usually `service solr restart` depending on solr/OS)
+
+8\. Run ReExportAll to update dataset metadata exports. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/5.14/admin/metadataexport.html#batch-exports-through-the-api).
+
+
 ## Backward Incompatibilities
 - Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version
 - Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility,
diff --git a/doc/release-notes/9002_allow_direct_upload_setting.md b/doc/release-notes/9002_allow_direct_upload_setting.md
deleted file mode 100644
index 1e76ed4ad47..00000000000
--- a/doc/release-notes/9002_allow_direct_upload_setting.md
+++ /dev/null
@@ -1,5 +0,0 @@
-A Dataverse installation can be now be configured to allow out-of-band upload by setting the `dataverse.files.<id>.upload-out-of-band` JVM option to `true`.
-
-By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
-
-With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.

From a78213633e6f5bf345d1aedf4328eee5ee231ffb Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 7 Dec 2023 10:43:40 -0500
Subject: [PATCH 905/922] #10151 remove notes previously incorporated

---
 .../9547-validation-for-geospatial-metadata.md     |  9 ---------
 doc/release-notes/9859-ORE and Bag updates.md      | 14 --------------
 2 files changed, 23 deletions(-)
 delete mode 100644 doc/release-notes/9547-validation-for-geospatial-metadata.md
 delete mode 100644 doc/release-notes/9859-ORE and Bag updates.md

diff --git a/doc/release-notes/9547-validation-for-geospatial-metadata.md b/doc/release-notes/9547-validation-for-geospatial-metadata.md
deleted file mode 100644
index a44e1a3732b..00000000000
--- a/doc/release-notes/9547-validation-for-geospatial-metadata.md
+++ /dev/null
@@ -1,9 +0,0 @@
-Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. (issue 9547). This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed.
-
-For the "upgrade" steps section:
-
-Update Geospatial Metadata Block
-
-- `wget https://github.com/IQSS/dataverse/releases/download/v6.1/geospatial.tsv`
-- `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file @geospatial.tsv`
-
diff --git a/doc/release-notes/9859-ORE and Bag updates.md b/doc/release-notes/9859-ORE and Bag updates.md
deleted file mode 100644
index dd3ae3bbbe1..00000000000
--- a/doc/release-notes/9859-ORE and Bag updates.md	
+++ /dev/null
@@ -1,14 +0,0 @@
-Dataverse's OAI_ORE Metadata Export format and archival BagIT exports 
-(which include the OAI-ORE metadata export file) have been updated to include 
-information about the dataset version state, e.g. RELEASED or DEACCESSIONED 
-and to indicate which version of Dataverse was used to create the archival Bag.
-As part of the latter, the current OAI_ORE Metadata format has been given a 1.0.0 
-version designation and it is expected that any future changes to the OAI_ORE export
-format will result in a version change and that tools such as DVUploader that can
-recreate datasets from archival Bags will start indicating which version(s) of the 
-OAI_ORE format they can read.
-
-Dataverse installations that have been using archival Bags may wish to update any
-existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
-[archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
-to generate updated versions.
\ No newline at end of file

From b517f6e0fca1802faa4455522a72e711963714ba Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Thu, 7 Dec 2023 10:53:07 -0500
Subject: [PATCH 906/922] #10151 S3 test notes

---
 doc/release-notes/6.1-release-notes.md | 2 ++
 doc/release-notes/6783-s3-tests.md     | 3 ---
 2 files changed, 2 insertions(+), 3 deletions(-)
 delete mode 100644 doc/release-notes/6783-s3-tests.md

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 9a35a31a734..375717ab9c9 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -163,6 +163,8 @@ For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.ht
 - `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
 also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
 - As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
+- Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT.
+- In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.
 
 ## OpenID Connect Authentication Provider Improvements
 
diff --git a/doc/release-notes/6783-s3-tests.md b/doc/release-notes/6783-s3-tests.md
deleted file mode 100644
index 1b9bb400cc6..00000000000
--- a/doc/release-notes/6783-s3-tests.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT.
-
-In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.

From 07a8659b60acdb766fb5a4742cf4ac4537e34615 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Fri, 8 Dec 2023 14:24:24 -0500
Subject: [PATCH 907/922] #10151 remove duplicate release note

out of band setting previously added
---
 doc/release-notes/6.1-release-notes.md | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 375717ab9c9..b6bb7d8b806 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -51,14 +51,6 @@ Previously, customization was possible by editing `Bundle.properties` but this i
 
 For details, see https://dataverse-guide--10122.org.readthedocs.build/en/10122/installation/config.html#bag-info-txt
 
-### Direct Upload setting added
-A Dataverse installation can be now be configured to allow out-of-band upload by setting the `dataverse.files.<id>.upload-out-of-band` JVM option to `true`.
-
-By default, Dataverse supports uploading files via the [add a file to a dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
-
-With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
-
-
 ### Improvements in the dataset versions API
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
 - a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output

From ed5b0dbde90fd4b8592aa2bdce7ae205482063c8 Mon Sep 17 00:00:00 2001
From: qqmyers <jim.myers@computer.org>
Date: Fri, 8 Dec 2023 15:44:18 -0500
Subject: [PATCH 908/922] Apply suggestions from code review

Co-authored-by: Philip Durbin <philipdurbin@gmail.com>
---
 doc/release-notes/10162-globus-support.md                | 2 +-
 doc/sphinx-guides/source/developers/big-data-support.rst | 4 ++--
 doc/sphinx-guides/source/developers/globus-api.rst       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/release-notes/10162-globus-support.md b/doc/release-notes/10162-globus-support.md
index 7bc3990f840..60670b5b101 100644
--- a/doc/release-notes/10162-globus-support.md
+++ b/doc/release-notes/10162-globus-support.md
@@ -1,7 +1,7 @@
 Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible, 
 and for referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. 
 Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/latest/developers/big-data-support.html#big-data-support)
-- Globus functionality remains 'experimental'/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incomatibilities.
+- Globus functionality remains 'experimental'/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incompatibilities.
 - The functionality is configured per store and replaces the previous single-S3-Connector-per-Dataverse-instance model
 - Adding files to a dataset, and accessing files is supported via the Dataverse user interface through a separate [dataverse-globus app](https://github.com/scholarsportal/dataverse-globus)
 - The functionality is also accessible via APIs (combining calls to the Dataverse and Globus APIs)
diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
index fe49f9f6150..8d891e63317 100644
--- a/doc/sphinx-guides/source/developers/big-data-support.rst
+++ b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -149,7 +149,7 @@ Globus File Transfer
 
 Note: Globus file transfer is still experimental but feedback is welcome! See :ref:`support`.
 
-Users can transfer files via `Globus <ttps://www.globus.org>`_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s) 
+Users can transfer files via `Globus <https://www.globus.org>`_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s) 
 and a community-developed `dataverse-globus <https://github.com/scholarsportal/dataverse-globus>`_ app has been properly installed and configured.
 
 Globus endpoints can be in a variety of places, from data centers to personal computers. 
@@ -168,7 +168,7 @@ Dataverse-managed endpoints must be Globus 'guest collections' hosted on either
 S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint.
 Users will need a Globus account, which can be obtained via their institution or directly from Globus (at no cost).
 
-With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionlity related to ingest, previews, fixity hash validation, etc. are not available. (Using the S3-based endpoint, Dataverse has access via S3 and all functionlity normally associated with direct uploads to S3 is available.)
+With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. (Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available.)
 
 For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials. 
 Users will need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described in the dataset's Terms of Use and Access).
diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst
index 5b2b6982866..37d80d0a6cd 100644
--- a/doc/sphinx-guides/source/developers/globus-api.rst
+++ b/doc/sphinx-guides/source/developers/globus-api.rst
@@ -71,7 +71,7 @@ The getDatasetMetadata and getFileListing URLs are just signed versions of the s
 If called for a dataset using a store that is configured with a remote Globus endpoint(s), the return response is similar but the response includes a
 the "managed" parameter will be false, the "endpoint" parameter is replaced with a JSON array of "referenceEndpointsWithPaths" and the
 requestGlobusTransferPaths and addGlobusFiles URLs are replaced with ones for requestGlobusReferencePaths and addFiles. All of these calls are
-describe further below.
+described further below.
 
 The call to set up for a transfer out (download) is similar:
 

From 1d668970df1562c3cbc85d60be2abc55d8a96572 Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Fri, 8 Dec 2023 15:56:27 -0500
Subject: [PATCH 909/922] #10151 standard guide links

---
 doc/release-notes/6.1-release-notes.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index b6bb7d8b806..24194a02026 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -49,14 +49,14 @@ Organization-Email: support@dataverse.harvard.edu
 
 Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.
 
-For details, see https://dataverse-guide--10122.org.readthedocs.build/en/10122/installation/config.html#bag-info-txt
+For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
 
 ### Improvements in the dataset versions API
 - optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
 - a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
 - when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files.
 
-This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/9763-lookup-optimizations/api/native-api.html#dataset-versions-api) section of the Guide.
+This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/6.1/api/native-api.html#dataset-versions-api) section of the Guide.
 
 ### The following API endpoints have been added:
 - deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
@@ -128,13 +128,13 @@ Dataverse installations that have been using archival Bags may wish to update an
 existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
 [archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
 to generate updated versions.
-- There is now a Markdown (.md) previewer: https://dataverse-guide--9986.org.readthedocs.build/en/9986/user/dataset-management.html#file-previews
+- There is now a Markdown (.md) previewer: https://guides.dataverse.org/en/6.1/user/dataset-management.html#file-previews
 - This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
   - To fix #9952, we surround the license info with `<` and `>`.
   - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
   - To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
 
-- We have started maintaining an API changelog: https://dataverse-guide--10127.org.readthedocs.build/en/10127/api/changelog.html
+- We have started maintaining an API changelog: https://guides.dataverse.org/en/6.1/api/changelog.html
 See also #10060.
 
 - Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)

From 85206de08acb6a8373199fb0d4eec2768cb6763d Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 8 Dec 2023 15:59:21 -0500
Subject: [PATCH 910/922] simply API changelog to be about breaking changes
 only #10151

---
 doc/release-notes/6.1-release-notes.md     |  2 +-
 doc/sphinx-guides/source/api/changelog.rst | 19 +++++--------------
 2 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 24194a02026..a3b04749d68 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -134,7 +134,7 @@ to generate updated versions.
   - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
   - To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
 
-- We have started maintaining an API changelog: https://guides.dataverse.org/en/6.1/api/changelog.html
+- We have started maintaining an API changelog of breaking changes: https://guides.dataverse.org/en/6.1/api/changelog.html
 See also #10060.
 
 - Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
index 910134e14f3..20225b99b5c 100644
--- a/doc/sphinx-guides/source/api/changelog.rst
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -1,5 +1,7 @@
-API Changelog 
-=============
+API Changelog (Breaking Changes)
+================================
+
+This API changelog is experimental and we would love feedback on its usefulness. Its primary purpose is to inform API developers of any breaking changes. (We try not ship any backward incompatible changes, but it happens.) To see a list of new APIs and backward-compatible changes to existing API, please see each version's release notes at https://github.com/IQSS/dataverse/releases
 
 .. contents:: |toctitle|
     :local:
@@ -8,20 +10,9 @@ API Changelog
 v6.1
 ----
 
-New
-~~~
-- **/api/dataverses/{id}/datasetSchema**: See :ref:`get-dataset-json-schema`.
-- **/api/dataverses/{id}/validateDatasetJson**: See :ref:`validate-dataset-json`.
-- **/api/admin/clearThumbnailFailureFlag**: See :ref:`thumbnail_reset`.
-- **/api/admin/downloadTmpFile**: See :ref:`download-file-from-tmp`.
-
-Changes
-~~~~~~~
-- **/api/datasets/{id}/versions/{versionId}/citation**: This endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation. See :ref:`get-citation`.
+- The metadata field "Alternative Title" now supports multiple values so you must pass an array rather than a string when populating that field via API. See https://github.com/IQSS/dataverse/pull/9440
 
 v6.0
 ----
 
-Changes
-~~~~~~~
 - **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch. See :doc:`dataaccess`.

From 0cd87d167211ee6bc047de3cba3e79acfb520e28 Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Fri, 8 Dec 2023 16:37:03 -0500
Subject: [PATCH 911/922] address Review comments

---
 .../source/admin/integrations.rst             | 12 ++++++++
 doc/sphinx-guides/source/api/intro.rst        |  4 +++
 .../source/developers/globus-api.rst          |  6 ++--
 .../source/installation/config.rst            |  9 +++---
 .../edu/harvard/iq/dataverse/DatasetPage.java | 29 -------------------
 .../AbstractRemoteOverlayAccessIO.java        |  2 +-
 6 files changed, 25 insertions(+), 37 deletions(-)

diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst
index 9a24cf0715c..db566106b49 100644
--- a/doc/sphinx-guides/source/admin/integrations.rst
+++ b/doc/sphinx-guides/source/admin/integrations.rst
@@ -121,6 +121,18 @@ Its goal is to make the dashboard adjustable for a Dataverse installation's need
 
 The integrations dashboard is currently in development. A preview and more information can be found at: `rdm-integration GitHub repository <https://github.com/libis/rdm-integration>`_
 
+Globus
+++++++
+
+Globus transfer uses an efficient transfer mechanism and has additional features that make it suitable for large files and large numbers of files:
+
+* robust file transfer capable of restarting after network or endpoint failures
+* third-party transfer, which enables a user accessing a Dataverse installation in their desktop browser to initiate transfer of their files from a remote endpoint (i.e. on a local high-performance computing cluster), directly to an S3 store managed by the Dataverse installation
+
+Users can transfer files via `Globus <https://www.globus.org>`_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s) 
+and a community-developed `dataverse-globus <https://github.com/scholarsportal/dataverse-globus>`_ app has been properly installed and configured.
+
+
 Embedding Data on Websites
 --------------------------
 
diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst
index 6c61bb8c20d..8eb11798dd7 100755
--- a/doc/sphinx-guides/source/api/intro.rst
+++ b/doc/sphinx-guides/source/api/intro.rst
@@ -187,6 +187,10 @@ Lists of Dataverse APIs
   - Files
   - etc.
 
+- :doc:`/developers/dataset-semantic-metadata-api`: For creating, reading, editing, and deleting dataset metadata using JSON-LD.
+- :doc:`/developers/dataset-migration-api`: For migrating datasets from other repositories while retaining the original persistent identifiers and publication date.
+- :doc:`/developers/s3-direct-upload-api`: For the transfer of larger files/larger numbers of files directly to an S3 bucket managed by Dataverse. 
+- :doc:`/developers/globus-api`: For the Globus transfer of larger files/larger numbers of files directly via Globus endpoints managed by Dataverse or referencing files in remote endpoints. 
 - :doc:`metrics`: For query statistics about usage of a Dataverse installation.
 - :doc:`sword`: For depositing data using a standards-based approach rather than the :doc:`native-api`.
 
diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst
index 37d80d0a6cd..de9df06a798 100644
--- a/doc/sphinx-guides/source/developers/globus-api.rst
+++ b/doc/sphinx-guides/source/developers/globus-api.rst
@@ -160,11 +160,11 @@ In the managed case, once a Globus transfer has been initiated a final API call
   export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
   export SERVER_URL=https://demo.dataverse.org
   export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
-  export JSON_DATA="{"taskIdentifier":"3f530302-6c48-11ee-8428-378be0d9c521", \
+  export JSON_DATA='{"taskIdentifier":"3f530302-6c48-11ee-8428-378be0d9c521", \
                     "files": [{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b3972213f-f6b5c2221423", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "1234"}}, \
-                    {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b39722140-50eb7d3c5ece", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "2345"}}]}"
+                    {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b39722140-50eb7d3c5ece", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "2345"}}]}'
 
-  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:multipart/form-data" -X POST "$SERVER_URL/api/datasets/:persistentId/addGlobusFiles -F "jsonData=$JSON_DATA""
+  curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:multipart/form-data" -X POST "$SERVER_URL/api/datasets/:persistentId/addGlobusFiles -F "jsonData=$JSON_DATA"
 
 Note that the mimetype is multipart/form-data, matching the /addFiles API call. ALso note that the API_TOKEN is not needed when using a signed URL.
 
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index 4540219fc7c..f6c05a3bde8 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -499,8 +499,8 @@ Logging & Slow Performance
 
 .. _file-storage:
 
-File Storage: Using a Local Filesystem and/or Swift and/or Object Stores and/or Trusted Remote Stores and/or Globus Stores
---------------------------------------------------------------------------------------------------------------------------
+File Storage
+------------
 
 By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara6/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.\<id\>.directory`` JVM option described below.
 
@@ -999,7 +999,8 @@ See :doc:`/developers/big-data-support` for additional information on how to use
 In addition to having the type "globus" and requiring a label, Globus Stores share many options with Trusted Remote Stores and options to specify and access a Globus endpoint(s). As with Remote Stores, Globus Stores also use a baseStore - a file, s3, or swift store that can be used to store additional ancillary dataset files (e.g. metadata exports, thumbnails, auxiliary files, etc.).
 These and other available options are described in the table below.
 
-There are two types of Globus stores
+There are two types of Globus stores:
+
 - managed - where Dataverse manages the Globus endpoint, deciding where transferred files are stored and managing access control for users transferring files to/from Dataverse
 - remote - where Dataverse references files that remain on trusted remote Globus endpoints
 
@@ -1024,7 +1025,7 @@ Once you have configured a globus store, it is recommended that you install the
     dataverse.files.<id>.globus-token                        <?>                 A Globus token (base64 endcoded <Globus user id>:<Credential> 
                                                                                  for a managed store) - using a microprofile alias is recommended            (none)
     dataverse.files.<id>.reference-endpoints-with-basepaths  <?>                 A comma separated list of *remote* trusted Globus endpoint id/<basePath>s   (none)
-    dataverse.files.<id>.files-not-accessible-by-dataverse   ``true``/``false``  Should be true for S3 Connector-based *managed* stores                      ``false``
+    dataverse.files.<id>.files-not-accessible-by-dataverse   ``true``/``false``  Should be false for S3 Connector-based *managed* stores, true for others    ``false``
     
     =======================================================  ==================  ==========================================================================  ===================
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
index 9c7d599ba33..b79f387f20b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java
@@ -5312,36 +5312,7 @@ public boolean isFileAccessRequestMultiButtonEnabled(){
         }
         return false;
     }
-/* These appear to be unused - toDo - delete
-    private Boolean downloadButtonAllEnabled = null;
 
-    public boolean isDownloadAllButtonEnabled() {
-
-        if (downloadButtonAllEnabled == null) {
-            for (FileMetadata fmd : workingVersion.getFileMetadatas()) {
-                if (!this.fileDownloadHelper.canDownloadFile(fmd)) {
-                    downloadButtonAllEnabled = false;
-                    break;
-                }
-            }
-            downloadButtonAllEnabled = true;
-        }
-        return downloadButtonAllEnabled;
-    }
-
-    public boolean isDownloadSelectedButtonEnabled(){
-
-        if( this.selectedFiles == null || this.selectedFiles.isEmpty() ){
-            return false;
-        }
-        for (FileMetadata fmd : this.selectedFiles){
-            if (this.fileDownloadHelper.canDownloadFile(fmd)){
-                return true;
-            }
-        }
-        return false;
-    }
-*/
     public boolean isFileAccessRequestMultiSignUpButtonRequired(){
         if (isSessionUserAuthenticated()){
             return false;
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
index 6c26502acfa..10ff68a56f3 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/AbstractRemoteOverlayAccessIO.java
@@ -50,7 +50,7 @@ public abstract class AbstractRemoteOverlayAccessIO<T extends DvObject> extends
     protected static final String REMOTE_STORE_URL = "remote-store-url";
     
     // Whether Dataverse can access the file bytes
-    //Currently True for the Globus store when using the S3Connector, and Remote Stores like simple web servers where the URLs resolve to the actual file bits
+    // Currently False only for the Globus store when using the S3Connector, and Remote Stores like simple web servers where the URLs resolve to the actual file bits
     static final String FILES_NOT_ACCESSIBLE_BY_DATAVERSE = "files-not-accessible-by-dataverse";
 
     protected StorageIO<DvObject> baseStore = null;

From ca706662cd9f19b36d31530cf2747d810923ca3e Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 11 Dec 2023 11:06:36 -0500
Subject: [PATCH 912/922] bug fix - allowing S3 w/Globus config to work for
 download

---
 .../iq/dataverse/dataaccess/GlobusAccessibleStore.java      | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
index e4d062f0619..8bed60d8302 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusAccessibleStore.java
@@ -65,7 +65,11 @@ public static String getGlobusToken(String storeId) {
     }
     
     public static boolean isGlobusAccessible(String storeId) {
-        if(StorageIO.getConfigParamForDriver(storeId, StorageIO.TYPE).equals(DataAccess.GLOBUS)) {
+        String type = StorageIO.getConfigParamForDriver(storeId, StorageIO.TYPE);
+        if (type.equals(DataAccess.GLOBUS)) {
+            return true;
+        } else if (type.equals(DataAccess.S3)
+                && StorageIO.getConfigParamForDriver(storeId, TRANSFER_ENDPOINT_WITH_BASEPATH) != null) {
             return true;
         }
         return false;

From 09a227b30a2b5da05829297a9173952596e2df9c Mon Sep 17 00:00:00 2001
From: qqmyers <qqmyers@hotmail.com>
Date: Mon, 11 Dec 2023 11:12:04 -0500
Subject: [PATCH 913/922] Change docs tp make clear that an S3 store can be
 used

---
 doc/sphinx-guides/source/installation/config.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index f6c05a3bde8..a7d7905ca4a 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1004,10 +1004,10 @@ There are two types of Globus stores:
 - managed - where Dataverse manages the Globus endpoint, deciding where transferred files are stored and managing access control for users transferring files to/from Dataverse
 - remote - where Dataverse references files that remain on trusted remote Globus endpoints
 
-For managed stores, there are two variants, connecting to standard/file-based Globus endpoints and to endpoints using an underlying S3 store via the Globus S3 Connector.
+A managed Globus store connects to standard/file-based Globus endpoint. It is also possible to configure an S3 store as a managed store, if the managed endpoint uses an underlying S3 store via the Globus S3 Connector.
 With the former, Dataverse has no direct access to the file contents and functionality related to ingest, fixity hash validation, etc. are not available. With the latter, Dataverse can access files internally via S3 and the functionality supported is similar to that when using S3 direct upload. 
 
-Once you have configured a globus store, it is recommended that you install the `dataverse-globus app <https://github.com/scholarsportal/dataverse-globus>`_ to allow transfers in/out of Dataverse to be initated via the Dataverse user interface. Alternately, you can point your users to the :doc:`/developers/globus-api` for information about API support.
+Once you have configured a globus store, or configured an S3 store for Globus access, it is recommended that you install the `dataverse-globus app <https://github.com/scholarsportal/dataverse-globus>`_ to allow transfers in/out of Dataverse to be initated via the Dataverse user interface. Alternately, you can point your users to the :doc:`/developers/globus-api` for information about API support.
 
 .. table::
     :align: left

From 44bd5b7fb6d697d356d857a73847e1637aaa5763 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Mon, 11 Dec 2023 11:19:46 -0500
Subject: [PATCH 914/922] add perf test results

---
 doc/release-notes/6.1-release-notes.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index a3b04749d68..b03a7a62baa 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -258,7 +258,7 @@ In the following commands we assume that Payara 6 is installed in `/usr/local/pa
 
 - `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
 
-7\. Upate Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with  custom metadata blocks  (7b).
+7\. Update Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with  custom metadata blocks  (7b).
 
 7a\. For installations without custom or experimental metadata blocks:
 
@@ -298,6 +298,10 @@ OR, alternatively, you can edit the following line in your schema.xml by hand as
 
 For the complete list of code changes in this release, see the [6.1 Milestone](https://github.com/IQSS/dataverse/milestone/110?closed=1) in GitHub.
 
+## Performance Testing Results
+The results of performance testing can be found here:
+https://docs.google.com/spreadsheets/d/1lwPlifvgu3-X_6xLwq6Zr6sCOervr1mV_InHIWjh5KA/edit#gid=0
+
 ## Getting Help
 
 For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.

From 173b8a7a067b392de8e1c900c3e1d9eb806c71d6 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Mon, 11 Dec 2023 11:25:44 -0500
Subject: [PATCH 915/922] fix backward comp Alternative Title

---
 doc/release-notes/6.1-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index b03a7a62baa..5bc0df4640c 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -291,7 +291,7 @@ OR, alternatively, you can edit the following line in your schema.xml by hand as
 
 
 ## Backward Incompatibilities
-- Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version
+- Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version. Alternative Title must now be passed as an array of strings rather than a single string ([alt title])
 - Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility,
 
 ## Complete List of Changes

From 1959f2ff22d9bbc4290a586fc49f1f49eccdbd04 Mon Sep 17 00:00:00 2001
From: Steven Winship <steven_winship@iq.harvard.edu>
Date: Mon, 11 Dec 2023 11:29:24 -0500
Subject: [PATCH 916/922] removed unneeded header

---
 doc/release-notes/6.1-release-notes.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 5bc0df4640c..6d3d1912f81 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -6,11 +6,8 @@ This release brings new features, enhancements, and bug fixes to the Dataverse s
 Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
 
 ## Release Highlights (Major Upgrades, Breaking Changes)
-
 This release contains major upgrades to core components. Detailed upgrade instructions can be found below.
 
-## Detailed Release Highlights, New Features and Use Case Scenarios
-
 ### Optional support for guestbooks to appear when files access is requested rather than after access has been granted and a download is started
 Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
   The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions.

From a4e25e17155896ae5c335ea8169229f248eaf22b Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Dec 2023 12:15:56 -0500
Subject: [PATCH 917/922] reorg 6.1 release notes, add globus #10151

---
 doc/release-notes/6.1-release-notes.md | 262 +++++++++++++------------
 1 file changed, 137 insertions(+), 125 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 6d3d1912f81..475d4fc0887 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -5,57 +5,96 @@ Please note: To read these instructions in full, please go to https://github.com
 This release brings new features, enhancements, and bug fixes to the Dataverse software.
 Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
 
-## Release Highlights (Major Upgrades, Breaking Changes)
-This release contains major upgrades to core components. Detailed upgrade instructions can be found below.
+## Release highlights
 
-### Optional support for guestbooks to appear when files access is requested rather than after access has been granted and a download is started
-Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
-  The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions.
+### Guestbook at request
+
+Dataverse can now be configured (via the `dataverse.files.guestbook-at-request` option) to display any configured guestbook to users when they request restricted files (new functionality) or when they download files (previous behavior).
+
+The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default, showing guestbooks when files are downloaded, remains as it was in prior Dataverse versions.
+
+### Collection-level storage quotas
+
+This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
+Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of Dataverse 6.1.
+
+### Globus support
+
+Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible and for the case of referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. Please note:
+
+- Globus functionality remains experimental/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incompatibilities.
+- The functionality is configured per store and replaces the previous single-S3-Connector-per-Dataverse-instance model.
+- Adding files to a dataset, and accessing files is supported via the Dataverse user interface through a separate [dataverse-globus app](https://github.com/scholarsportal/dataverse-globus).
+- The functionality is also accessible via APIs (combining calls to the Dataverse and Globus APIs)
+
+Backward incompatibilities:
+- The configuration for use of a Globus S3 Connector has changed and is aligned with the standard store configuration mechanism
+- The new functionality is incompatible with older versions of the globus-dataverse app and the Globus-related functionality in the UI will only function correctly if a Dataverse 6.1 compatible version of the dataverse-globus app is configured.
+
+New JVM options:
+- A new "globus" store type and associated store-related options have been added. These are described in the [File Storage Options section of the Dataverse Guides](https://guides.dataverse.org/en/latest/installation/config.html#file-storage-using-a-local-filesystem-and-or-swift-and-or-object-stores-and-or-trusted-remote-stores).
+- dataverse.files.globus-cache-maxage - specifies the number of minutes Dataverse will wait between an initial request for a file transfer occurs and when that transfer must begin.
+
+Obsolete Settings: the :GlobusBasicToken, :GlobusEndpoint, and :GlobusStores settings are no longer used
+
+Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/6.1/developers/big-data-support.html#big-data-support)
+
+### Alternative Title now allows multiple values
+
+Alternative Title now allows multiples. Note that JSON used to create a dataset with an Alternate Title must be changed. See "Backward incompatibilities" below for details.
+
+### External tools: configure tools now available at the dataset level
+
+Read/write "configure" tools (a type of external tool) are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
+
+### S3 out-of-band upload
 
-### Dataverse installation can be now be configured to allow out-of-band upload
 In some situations, direct upload might not work from the UI, e.g., when s3 storage is not accessible from the internet. This pull request adds an option to [allow direct uploads via API only](https://github.com/IQSS/dataverse/pull/9003). This way, a third party application can use direct upload from within the internal network, while there is no direct download available to the users via UI.
 By default, Dataverse supports uploading files via the [add a file to a dataset](https://guides.dataverse.org/en/6.1/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
 With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
-### Alternative Title is made repeatable.
-- One will need to update database with updated citation block.
-  `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
-- One will also need to update Solr schema:
- Change in "alternativeTitle" field  multiValued="true" in `/usr/local/solr/solr-9.3.0/server/solr/collection1/conf/schema.xml`
- Reload Solr schema: `curl "http://localhost:8983/solr/admin/cores?action=RELOAD&core=collection1"`
+### JSON Schema for datasets
 
-Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version since value of alternative title has changed from simple string to an array.
-For example, instead "value": "Alternative Title", the value can be "value": ["Alternative Title1", "Alternative Title2"]
+Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
 
-### Collection Storage Size Quota Support
--This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
-Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of 6.1
+### OpenID Connect authentication provider improvements
 
-### BagIT Export Configurations Updated
-For BagIT export, it is now possible to configure the following information in bag-info.txt...
+#### Using MicroProfile Config for provisioning
 
-Source-Organization: Harvard Dataverse
-Organization-Address: 1737 Cambridge Street, Cambridge, MA, USA
-Organization-Email: support@dataverse.harvard.edu
+With this release it is possible to provision a single OIDC-based authentication provider
+by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
 
-... using new JVM/MPCONFIG options:
+If you are using an external OIDC provider component as an identity management system and/or broker
+to other authentication providers such as Google, eduGain SAML and so on, this might make your
+life easier during instance setups and reconfiguration. You no longer need to generate the
+necessary JSON file.
 
-- dataverse.bagit.sourceorg.name
-- dataverse.bagit.sourceorg.address
-- dataverse.bagit.sourceorg.email
+#### Adding PKCE Support
 
-Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.
+[This PR adds PKCE support for OIDC providers](https://github.com/IQSS/dataverse/pull/9273)
+Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
+support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
 
-For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
+### Solr improvements
 
-### Improvements in the dataset versions API
-- optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
-- a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
-- when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files.
+As of this release, application-side support has been added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
 
-This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/6.1/api/native-api.html#dataset-versions-api) section of the Guide.
+Please see the "Installing Solr" section of the Installation Prerequisites guide.
+
+### New release of Dataverse Previewers (including a Markdown previewer)
+
+Version 1.4 of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended. Please note:
+
+- SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
+- Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
+- There is now a Markdown (.md) previewer: https://guides.dataverse.org/en/6.1/user/dataset-management.html#file-previews
+
+### New or improved APIs
+
+The development of a [new UI for Dataverse](https://github.com/IQSS/dataverse-frontend) is driving the addition or improvement of many APIs.
+
+#### New API endpoints
 
-### The following API endpoints have been added:
 - deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
 - /api/files/{id}/downloadCount
 - /api/files/{id}/dataTables 
@@ -71,7 +110,33 @@ This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/
 - validateDatasetJsonSchema (/api/dataverses/{id}/validateDatasetJson): Validate that a dataset JSON file is in proper format and contains the required elements and fields for a given dataverse collection.
 - downloadTmpFile (/api/admin/downloadTmpFile): For testing purposes, allows files to be downloaded from /tmp.
 
-### Extended the existing endpoints:
+#### Pagination of files in dataset versions
+
+- optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
+- a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
+- when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files.
+
+This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/6.1/api/native-api.html#dataset-versions-api) section of the Guide.
+
+
+#### DataFile API payload has been extended to include the following fields
+
+- tabularData: Boolean field to know if the DataFile is of tabular type
+- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner)
+- friendlyType: String
+
+#### The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination, ordering, and optional filtering
+
+- Access status: through the `accessStatus` query parameter, which supports the following values:
+    - Public
+    - Restricted
+    - EmbargoedThenRestricted
+    - EmbargoedThenPublic
+- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
+- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
+
+#### Additional improvements to existing API endpoints
+
 - getVersionFiles (/api/datasets/{id}/versions/{versionId}/files): Extended to support optional filtering by search text through the `searchText` query parameter. The search will be applied to the labels and descriptions of the dataset files. Added `tabularTagName` to return files to which the particular tabular tag has been added. Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files.
 - getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain file counts. Added support for filtering by optional criteria query parameter:
     - contentType
@@ -93,25 +158,21 @@ This parameter applies a filter criteria to the operation and supports the follo
     - Can delete the dataset draft
 - getDatasetVersionCitation (/api/datasets/{id}/versions/{versionId}/citation) endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation.
 
-### DataFile API payload has been extended to include the following fields:
-- tabularData: Boolean field to know if the DataFile is of tabular type
-- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner)
-- friendlyType: String
+### Improvements for developers
 
-### The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination, ordering, and optional filtering
-- Access status: through the `accessStatus` query parameter, which supports the following values:
-    - Public
-    - Restricted
-    - EmbargoedThenRestricted
-    - EmbargoedThenPublic
-- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
-- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
+- Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using Netbeans or IntelliJ IDEA Ultimate (with the Payara Platform Tools plugin). For details, see https://guides.dataverse.org/en/6.1/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools and [the thread](https://groups.google.com/g/dataverse-community/c/zNBDzSMF2Q0/m/Z-xS6fA2BgAJ) on the mailing list.
+- Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT.
+- In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.
+- With this release, we add a new type of testing to Dataverse: integration tests which are not end-to-end tests (like our API tests). Starting with OIDC authentication support, we test regularly on CI for working condition of both OIDC login options in UI and API.
+- The testing and development Keycloak realm has been updated with more users and compatibility with Keycloak 21.
+- The support for setting JVM options during testing has been improved for developers. You now may add the `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
+- As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
 
+## Major use cases and infrastructure enhancements
 
-### Misc
-- Configure tools are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
-- Dataverse can now be configured (via the dataverse.files.guestbook-at-request option) to display any configured guestbook to users when they request restricted file(s) or when they download files (the historic default).
-The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default - showing guestbooks when files are downloaded - remains as it was in prior Dataverse versions.
+Changes and fixes in this release not already mentioned above include:
+
+- Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. (issue #9547). This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed.
 - Dataverse's OAI_ORE Metadata Export format and archival BagIT exports
 (which include the OAI-ORE metadata export file) have been updated to include
 information about the dataset version state, e.g. RELEASED or DEACCESSIONED
@@ -125,68 +186,18 @@ Dataverse installations that have been using archival Bags may wish to update an
 existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
 [archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
 to generate updated versions.
-- There is now a Markdown (.md) previewer: https://guides.dataverse.org/en/6.1/user/dataset-management.html#file-previews
+- For BagIT export, it is now possible to configure the following information in bag-info.txt. (Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.) For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
+  - Source-Organization from `dataverse.bagit.sourceorg.name`.
+  - Organization-Address from `dataverse.bagit.sourceorg.address`.
+  - Organization-Email from `dataverse.bagit.sourceorg.address`.
 - This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
   - To fix #9952, we surround the license info with `<` and `>`.
   - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
   - To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
-
 - We have started maintaining an API changelog of breaking changes: https://guides.dataverse.org/en/6.1/api/changelog.html
 See also #10060.
 
-- Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
-- Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. (issue 9547). This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed.
-
-### Solr Improvements
-- As of this release application-side support is added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
-
-Please see the "Installing Solr" section of the Installation Prerequisites guide.
-
-
-### Development
-- Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using Netbeans or IntelliJ IDEA Ultimate (free educational licenses are available) and the Payara Platform Tools plugin.
-For details, see http://preview.guides.gdcc.io/en/develop/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools and [the thread](https://groups.google.com/g/dataverse-community/c/zNBDzSMF2Q0/m/Z-xS6fA2BgAJ) on the mailing list.
-- A new version of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended.
-  - SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
-  - Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
-- `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
-also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
-- As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
-- Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT.
-- In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.
-
-## OpenID Connect Authentication Provider Improvements
-
-### Using MicroProfile Config For Provisioning
-
-With this release it is possible to provision a single OIDC-based authentication provider
-by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
-
-If you are using an external OIDC provider component as an identity management system and/or broker
-to other authentication providers such as Google, eduGain SAML and so on, this might make your
-life easier during instance setups and reconfiguration. You no longer need to generate the
-necessary JSON file.
-
-### Adding PKCE Support
-[This PR adds PKCE support for OIDC providers](https://github.com/IQSS/dataverse/pull/9273)
-Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
-support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
-
-## Improved Testing
-
-With this release, we add a new type of testing to Dataverse: integration tests which are no end-to-end tests
-like our API tests. Starting with OIDC authentication support, we test regularly on CI for working condition
-of both OIDC login options in UI and API.
-
-The testing and development Keycloak realm has been updated with more users and compatibility with Keycloak 21.
-
-The support for setting JVM options during testing has been improved for developers. You now may add the
-`@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is
-also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
-
-As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
-
-## New Configuration Options
+## New configuration options
 
 - dataverse.auth.oidc.enabled
 - dataverse.auth.oidc.client-id
@@ -199,8 +210,24 @@ As part of these testing improvements, the code coverage report file for unit te
 - dataverse.auth.oidc.pkce.max-cache-size
 - dataverse.auth.oidc.pkce.max-cache-age
 - dataverse.files.{driverId}.upload-out-of-band
+- dataverse.files.globus-cache-maxage
 - dataverse.files.guestbook-at-request
 
+## Backward incompatibilities
+
+- Since Alternative Title is now repeatable, the JSON you send to create or edit a dataset must be an array rather than a simple string. For example, instead of "value": "Alternative Title", you must send "value": ["Alternative Title1", "Alternative Title2"]
+- Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility. See above for details.
+- For BagIT export, if you were configuring values in bag-info.txt using `Bundle.properties`, you must switch to the new JVM options `dataverse.bagit.sourceorg.name`, `dataverse.bagit.sourceorg.address`, and `dataverse.bagit.sourceorg.email`. For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
+- See "Globus support" above for backward incompatibilies specific to Globus.
+
+## Complete list of changes
+
+For the complete list of code changes in this release, see the [6.1 Milestone](https://github.com/IQSS/dataverse/milestone/110?closed=1) in GitHub.
+
+## Getting help
+
+For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
+
 ## Installation
 
 If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it!
@@ -209,7 +236,7 @@ Once you are in production, we would be delighted to update our [map of Datavers
 
 You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC).
 
-## Upgrade Instructions
+## Upgrade instructions
 Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
 
 These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
@@ -241,6 +268,8 @@ In the following commands we assume that Payara 6 is installed in `/usr/local/pa
 
 - `$PAYARA/bin/asadmin deploy dataverse-6.1.war`
 
+As noted above, deployment of the war file might take several minutes due a database migration script required for the new storage quotas feature.
+
 5\. Restart Payara
 
 - `service payara stop`
@@ -255,7 +284,7 @@ In the following commands we assume that Payara 6 is installed in `/usr/local/pa
 
 - `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
 
-7\. Update Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with  custom metadata blocks  (7b).
+7\. Upate Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with  custom metadata blocks  (7b).
 
 7a\. For installations without custom or experimental metadata blocks:
 
@@ -285,20 +314,3 @@ OR, alternatively, you can edit the following line in your schema.xml by hand as
 - Restart Solr instance (usually `service solr restart` depending on solr/OS)
 
 8\. Run ReExportAll to update dataset metadata exports. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/5.14/admin/metadataexport.html#batch-exports-through-the-api).
-
-
-## Backward Incompatibilities
-- Since Alternative Title is repeatable now, old JSON APIs would not be compatible with a new version. Alternative Title must now be passed as an array of strings rather than a single string ([alt title])
-- Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility,
-
-## Complete List of Changes
-
-For the complete list of code changes in this release, see the [6.1 Milestone](https://github.com/IQSS/dataverse/milestone/110?closed=1) in GitHub.
-
-## Performance Testing Results
-The results of performance testing can be found here:
-https://docs.google.com/spreadsheets/d/1lwPlifvgu3-X_6xLwq6Zr6sCOervr1mV_InHIWjh5KA/edit#gid=0
-
-## Getting Help
-
-For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.

From 011b9291e6f694631d237bd047c3a170e6e93a2e Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Dec 2023 13:58:08 -0500
Subject: [PATCH 918/922] remove globus snippet (already added) #10151

---
 doc/release-notes/10162-globus-support.md | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 doc/release-notes/10162-globus-support.md

diff --git a/doc/release-notes/10162-globus-support.md b/doc/release-notes/10162-globus-support.md
deleted file mode 100644
index 60670b5b101..00000000000
--- a/doc/release-notes/10162-globus-support.md
+++ /dev/null
@@ -1,19 +0,0 @@
-Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible, 
-and for referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. 
-Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/latest/developers/big-data-support.html#big-data-support)
-- Globus functionality remains 'experimental'/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incompatibilities.
-- The functionality is configured per store and replaces the previous single-S3-Connector-per-Dataverse-instance model
-- Adding files to a dataset, and accessing files is supported via the Dataverse user interface through a separate [dataverse-globus app](https://github.com/scholarsportal/dataverse-globus)
-- The functionality is also accessible via APIs (combining calls to the Dataverse and Globus APIs)
-
-Backward Incompatibilities:
-- The configuration for use of a Globus S3 Connector has changed and is aligned with the standard store configuration mechanism
-- The new functionality is incompatible with older versions of the globus-dataverse app and the Globus-related functionality in the UI will only function correctly if a Dataverse 6.1 compatible version of the dataverse-globus app is configured.
-
-New JVM Options:
-- A new 'globus' store type and associated store-related options have been added. These are described in the [File Storage Options section of the Dataverse Guides](https://guides.dataverse.org/en/latest/installation/config.html#file-storage-using-a-local-filesystem-and-or-swift-and-or-object-stores-and-or-trusted-remote-stores).
-- dataverse.files.globus-cache-maxage - specifies the number of minutes Dataverse will wait between an initial request for a file transfer occurs and when that transfer must begin.
-
-
-
-Obsolete Settings: the :GlobusBasicToken, :GlobusEndpoint, and :GlobusStores settings are no longer used

From 3e32f42959dce41e9c21c9e2285fdf719b048dc0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Dec 2023 14:57:52 -0500
Subject: [PATCH 919/922] link to guides in more places, other tweaks #10151

---
 doc/release-notes/6.1-release-notes.md | 43 +++++++++++++-------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 475d4fc0887..fab11ce4959 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -13,12 +13,17 @@ Dataverse can now be configured (via the `dataverse.files.guestbook-at-request`
 
 The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default, showing guestbooks when files are downloaded, remains as it was in prior Dataverse versions.
 
+For details, see [dataverse.files.guestbook-at-request](https://guides.dataverse.org/en/6.1/installation/config.html#dataverse-files-guestbook-at-request) and PR #9599.
+
 ### Collection-level storage quotas
 
 This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
+
 Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of Dataverse 6.1.
 
-### Globus support
+For details, see [Storage Quotas for Collections](https://guides.dataverse.org/en/6.1/admin/collectionquotas.html) in the Admin Guide.
+
+### Globus support (experimental), continued
 
 Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible and for the case of referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. Please note:
 
@@ -32,54 +37,50 @@ Backward incompatibilities:
 - The new functionality is incompatible with older versions of the globus-dataverse app and the Globus-related functionality in the UI will only function correctly if a Dataverse 6.1 compatible version of the dataverse-globus app is configured.
 
 New JVM options:
-- A new "globus" store type and associated store-related options have been added. These are described in the [File Storage Options section of the Dataverse Guides](https://guides.dataverse.org/en/latest/installation/config.html#file-storage-using-a-local-filesystem-and-or-swift-and-or-object-stores-and-or-trusted-remote-stores).
+- A new "globus" store type and associated store-related options have been added. These are described in the [File Storage](https://guides.dataverse.org/en/6.1/installation/config.html#file-storage) section of the Installation Guide.
 - dataverse.files.globus-cache-maxage - specifies the number of minutes Dataverse will wait between an initial request for a file transfer occurs and when that transfer must begin.
 
 Obsolete Settings: the :GlobusBasicToken, :GlobusEndpoint, and :GlobusStores settings are no longer used
 
-Further details can be found in the [Big Data Support section of the Dataverse Guides](https://guides.dataverse.org/en/6.1/developers/big-data-support.html#big-data-support)
+Further details can be found in the [Big Data Support](https://guides.dataverse.org/en/6.1/developers/big-data-support.html#big-data-support) section of the Developer Guide.
 
 ### Alternative Title now allows multiple values
 
-Alternative Title now allows multiples. Note that JSON used to create a dataset with an Alternate Title must be changed. See "Backward incompatibilities" below for details.
+Alternative Title now allows multiples. Note that JSON used to create a dataset with an Alternate Title must be changed. See "Backward incompatibilities" below and PR #9440 for details.
 
 ### External tools: configure tools now available at the dataset level
 
-Read/write "configure" tools (a type of external tool) are now available at the dataset level. They appear under the "Edit Dataset" menu. See also #9589.
+Read/write "configure" tools (a type of external tool) are now available at the dataset level. They appear under the "Edit Dataset" menu. See [External Tools](https://guides.dataverse.org/en/6.1/admin/external-tools.html#dataset-level-configure-tools) in the Admin Guide and PR #9925.
 
 ### S3 out-of-band upload
 
 In some situations, direct upload might not work from the UI, e.g., when s3 storage is not accessible from the internet. This pull request adds an option to [allow direct uploads via API only](https://github.com/IQSS/dataverse/pull/9003). This way, a third party application can use direct upload from within the internal network, while there is no direct download available to the users via UI.
 By default, Dataverse supports uploading files via the [add a file to a dataset](https://guides.dataverse.org/en/6.1/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
-With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://dataverse-guide--9003.org.readthedocs.build/en/9003/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://guides.dataverse.org/en/6.1/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://guides.dataverse.org/en/6.1/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
 
 ### JSON Schema for datasets
 
-Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. (Issue #9464 and #9465)
+Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.1/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide and PR #10109.
 
-### OpenID Connect authentication provider improvements
+### OpenID Connect (OIDC) improvements
 
 #### Using MicroProfile Config for provisioning
 
-With this release it is possible to provision a single OIDC-based authentication provider
-by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
+With this release it is possible to provision a single OIDC-based authentication provider by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
 
-If you are using an external OIDC provider component as an identity management system and/or broker
-to other authentication providers such as Google, eduGain SAML and so on, this might make your
-life easier during instance setups and reconfiguration. You no longer need to generate the
-necessary JSON file.
+If you are using an external OIDC provider component as an identity management system and/or broker to other authentication providers such as Google, eduGain SAML and so on, this might make your life easier during instance setups and reconfiguration. You no longer need to generate the necessary JSON file.
 
 #### Adding PKCE Support
 
-[This PR adds PKCE support for OIDC providers](https://github.com/IQSS/dataverse/pull/9273)
-Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable
-support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
+Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
+
+For both features, see the [OIDC](https://guides.dataverse.org/en/6.0/installation/oidc.html) section of the Installation Guide and PR #9273.
 
 ### Solr improvements
 
 As of this release, application-side support has been added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
 
-Please see the "Installing Solr" section of the Installation Prerequisites guide.
+Please see the [Installing Solr](https://guides.dataverse.org/en/6.1/installation/prerequisites.html#installing-solr) section of the Installation Guide.
 
 ### New release of Dataverse Previewers (including a Markdown previewer)
 
@@ -87,7 +88,7 @@ Version 1.4 of the standard Dataverse Previewers from https://github/com/gdcc/da
 
 - SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
 - Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
-- There is now a Markdown (.md) previewer: https://guides.dataverse.org/en/6.1/user/dataset-management.html#file-previews
+- There is now a [Markdown (.md)](https://guides.dataverse.org/en/6.1/user/dataset-management.html#file-previews) previewer.
 
 ### New or improved APIs
 
@@ -172,7 +173,7 @@ This parameter applies a filter criteria to the operation and supports the follo
 
 Changes and fixes in this release not already mentioned above include:
 
-- Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. (issue #9547). This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed.
+- Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed. See PR #10142.
 - Dataverse's OAI_ORE Metadata Export format and archival BagIT exports
 (which include the OAI-ORE metadata export file) have been updated to include
 information about the dataset version state, e.g. RELEASED or DEACCESSIONED
@@ -184,7 +185,7 @@ recreate datasets from archival Bags will start indicating which version(s) of t
 OAI_ORE format they can read.
 Dataverse installations that have been using archival Bags may wish to update any
 existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
-[archival Bag export API](https://guides.dataverse.org/en/latest/installation/config.html#bagit-export-api-calls)
+[archival Bag export API](https://guides.dataverse.org/en/6.1/installation/config.html#bagit-export-api-calls)
 to generate updated versions.
 - For BagIT export, it is now possible to configure the following information in bag-info.txt. (Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.) For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
   - Source-Organization from `dataverse.bagit.sourceorg.name`.

From 92a298da25c03822c848e5a43253f039193665f9 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Dec 2023 15:42:55 -0500
Subject: [PATCH 920/922] add missing new config options and sort #10151

---
 doc/release-notes/6.1-release-notes.md | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index fab11ce4959..1e09a207104 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -200,25 +200,28 @@ See also #10060.
 
 ## New configuration options
 
-- dataverse.auth.oidc.enabled
+- dataverse.auth.oidc.auth-server-url
 - dataverse.auth.oidc.client-id
 - dataverse.auth.oidc.client-secret
-- dataverse.auth.oidc.auth-server-url
+- dataverse.auth.oidc.enabled
 - dataverse.auth.oidc.pkce.enabled
+- dataverse.auth.oidc.pkce.max-cache-age
+- dataverse.auth.oidc.pkce.max-cache-size
 - dataverse.auth.oidc.pkce.method
-- dataverse.auth.oidc.title
 - dataverse.auth.oidc.subtitle
-- dataverse.auth.oidc.pkce.max-cache-size
-- dataverse.auth.oidc.pkce.max-cache-age
-- dataverse.files.{driverId}.upload-out-of-band
+- dataverse.auth.oidc.title
+- dataverse.bagit.sourceorg.address
+- dataverse.bagit.sourceorg.address
+- dataverse.bagit.sourceorg.name
 - dataverse.files.globus-cache-maxage
 - dataverse.files.guestbook-at-request
+- dataverse.files.{driverId}.upload-out-of-band
 
 ## Backward incompatibilities
 
 - Since Alternative Title is now repeatable, the JSON you send to create or edit a dataset must be an array rather than a simple string. For example, instead of "value": "Alternative Title", you must send "value": ["Alternative Title1", "Alternative Title2"]
 - Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility. See above for details.
-- For BagIT export, if you were configuring values in bag-info.txt using `Bundle.properties`, you must switch to the new JVM options `dataverse.bagit.sourceorg.name`, `dataverse.bagit.sourceorg.address`, and `dataverse.bagit.sourceorg.email`. For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
+- For BagIT export, if you were configuring values in bag-info.txt using `Bundle.properties`, you must switch to the new `dataverse.bagit` JVM options mentioned above. For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
 - See "Globus support" above for backward incompatibilies specific to Globus.
 
 ## Complete list of changes

From 80634c7a59d7bfce4ab0e871d80d34f446579123 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Dec 2023 15:54:30 -0500
Subject: [PATCH 921/922] address feedback from review #9919

---
 doc/sphinx-guides/source/developers/performance.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/developers/performance.rst b/doc/sphinx-guides/source/developers/performance.rst
index aa50cd6e40c..46c152f322e 100644
--- a/doc/sphinx-guides/source/developers/performance.rst
+++ b/doc/sphinx-guides/source/developers/performance.rst
@@ -116,12 +116,12 @@ We'd like to rate limit commands (CreateDataset, etc.) so that we can keep them
 Solr
 ~~~~
 
-While in the past Solr performance hasn't been much of a concern, in recent years we've noticed performance problems when Harvard Dataverse is under load. We are investigating in `#9635 <https://github.com/IQSS/dataverse/issues/9635>`_.
+While in the past Solr performance hasn't been much of a concern, in recent years we've noticed performance problems when Harvard Dataverse is under load. Improvements were made in `PR #10050 <https://github.com/IQSS/dataverse/pull/10050>`_, for example.
 
 Datasets with Large Numbers of Files or Versions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-We'd like to scale Dataverse to better handle large number of files or versions (`#9763 <https://github.com/IQSS/dataverse/issues/9763>`_).
+We'd like to scale Dataverse to better handle large number of files or versions. Progress was made in `PR #9883 <https://github.com/IQSS/dataverse/pull/9883>`_.
 
 Withstanding Bots
 ~~~~~~~~~~~~~~~~~
@@ -183,7 +183,7 @@ Most likely there is training available that is oriented toward performance. The
 Learn from the Community How They Monitor Performance
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Some members of the Dataverse community are likely users of newish tools like the ELK stack (Elasticsearch, Logstash, and Kibana), the TICK stack (Telegraph InfluxDB Chronograph and Kapacitor), GoAccess, Prometheus, Graphite, and more we haven't even heard of. In the :doc:`/admin/monitoring` section of the Admin Guide, we already encourage the community to share findings (, but we could dedicate time to this topic at our annual meeting or community calls.
+Some members of the Dataverse community are likely users of newish tools like the ELK stack (Elasticsearch, Logstash, and Kibana), the TICK stack (Telegraph InfluxDB Chronograph and Kapacitor), GoAccess, Prometheus, Graphite, and more we haven't even heard of. In the :doc:`/admin/monitoring` section of the Admin Guide, we already encourage the community to share findings, but we could dedicate time to this topic at our annual meeting or community calls.
 
 Teach the Community to Do Performance Testing
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 3d6343eca2846edca97e4d9699f3305fb7c19c62 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 11 Dec 2023 16:09:46 -0500
Subject: [PATCH 922/922] mention configurable docroot #10151

---
 doc/release-notes/6.1-release-notes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
index 1e09a207104..1279d09a023 100644
--- a/doc/release-notes/6.1-release-notes.md
+++ b/doc/release-notes/6.1-release-notes.md
@@ -195,6 +195,7 @@ to generate updated versions.
   - To fix #9952, we surround the license info with `<` and `>`.
   - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
   - To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
+- It's now possible to configure the docroot, which holds collection logos and more. See [dataverse.files.docroot](https://guides.dataverse.org/en/6.1/installation/config.html#dataverse-files-docroot) in the Installation Guide and PR #9819. 
 - We have started maintaining an API changelog of breaking changes: https://guides.dataverse.org/en/6.1/api/changelog.html
 See also #10060.
 
@@ -213,6 +214,7 @@ See also #10060.
 - dataverse.bagit.sourceorg.address
 - dataverse.bagit.sourceorg.address
 - dataverse.bagit.sourceorg.name
+- dataverse.files.docroot
 - dataverse.files.globus-cache-maxage
 - dataverse.files.guestbook-at-request
 - dataverse.files.{driverId}.upload-out-of-band