From d3de1fa8f0e4d29db512210bbb5eadedba14bf00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20Haarla=CC=88nder?= <haarlaender@mpdl.mpg.de>
Date: Mon, 23 May 2022 14:27:42 +0200
Subject: [PATCH 01/25] #IQSS/8726 better HTTP range request support

---
 src/main/java/edu/harvard/iq/dataverse/api/Access.java     | 7 +++++--
 .../edu/harvard/iq/dataverse/api/ApiBlockingFilter.java    | 3 ++-
 .../harvard/iq/dataverse/api/DownloadInstanceWriter.java   | 3 +++
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
index b2a8da3af4c..cfb30cc0753 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java
@@ -278,7 +278,7 @@ private DataFile findDataFileOrDieWrapper(String fileId){
     @Path("datafile/{fileId:.+}")
     @GET
     @Produces({"application/xml"})
-    public DownloadInstance datafile(@PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
+    public Response datafile(@PathParam("fileId") String fileId, @QueryParam("gbrecs") boolean gbrecs, @QueryParam("key") String apiToken, @Context UriInfo uriInfo, @Context HttpHeaders headers, @Context HttpServletResponse response) /*throws NotFoundException, ServiceUnavailableException, PermissionDeniedException, AuthorizationRequiredException*/ {
         
         // check first if there's a trailing slash, and chop it: 
         while (fileId.lastIndexOf('/') == fileId.length() - 1) {
@@ -423,7 +423,10 @@ public DownloadInstance datafile(@PathParam("fileId") String fileId, @QueryParam
         /* 
          * Provide some browser-friendly headers: (?)
          */
-        return downloadInstance;
+        if (headers.getRequestHeaders().containsKey("Range")) {
+            return Response.status(Response.Status.PARTIAL_CONTENT).entity(downloadInstance).build();
+        }
+        return Response.ok(downloadInstance).build();
     }
     
     
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java b/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java
index 6f7a1d876a1..6bf852d25f7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/ApiBlockingFilter.java
@@ -163,7 +163,8 @@ public void doFilter(ServletRequest sr, ServletResponse sr1, FilterChain fc) thr
             if (settingsSvc.isTrueForKey(SettingsServiceBean.Key.AllowCors, true )) {
                 ((HttpServletResponse) sr1).addHeader("Access-Control-Allow-Origin", "*");
                 ((HttpServletResponse) sr1).addHeader("Access-Control-Allow-Methods", "PUT, GET, POST, DELETE, OPTIONS");
-                ((HttpServletResponse) sr1).addHeader("Access-Control-Allow-Headers", "Accept, Content-Type, X-Dataverse-Key");
+                ((HttpServletResponse) sr1).addHeader("Access-Control-Allow-Headers", "Accept, Content-Type, X-Dataverse-Key, Range");
+                ((HttpServletResponse) sr1).addHeader("Access-Control-Expose-Headers", "Accept-Ranges, Content-Range, Content-Encoding");
             }
             fc.doFilter(sr, sr1);
         } catch ( ServletException se ) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
index 84a31959286..78fdb261d38 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java
@@ -434,6 +434,9 @@ public void writeTo(DownloadInstance di, Class<?> clazz, Type type, Annotation[]
                                 
                                 offset = ranges.get(0).getStart();
                                 leftToRead = rangeContentSize;
+                                httpHeaders.add("Accept-Ranges", "bytes");
+                                httpHeaders.add("Content-Range", "bytes "+offset+"-"+(offset+rangeContentSize-1)+"/"+contentSize);
+
                             }
                         } else {
                             // Content size unknown, must be a dynamically

From 320274ebd34edf983d8872ba773ec244adf14c4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20Haarla=CC=88nder?= <haarlaender@mpdl.mpg.de>
Date: Wed, 1 Jun 2022 09:51:20 +0200
Subject: [PATCH 02/25] #IQSS/8757 deactivate file restriction for
 PublicInstall

---
 src/main/webapp/editFilesFragment.xhtml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml
index 883949d0441..5013eacb159 100644
--- a/src/main/webapp/editFilesFragment.xhtml
+++ b/src/main/webapp/editFilesFragment.xhtml
@@ -440,7 +440,7 @@
                                                     <h:outputText value="#{bundle['file.delete']}"/> 
                                                 </p:commandLink>
                                             </li>
-                                            <li>
+                                            <li jsf:rendered="#{!settingsWrapper.publicInstall}">
                                                 <p:commandLink  onclick="if (!testFilesSelected2()) return false;"
                                                                 update="@([id$=editFileAccessPopup])" 
                                                                 oncomplete="PF('editFileAccessPopup').show();bind_bsui_components();">                                                               
@@ -449,7 +449,7 @@
                                                     <f:setPropertyActionListener target="#{EditDatafilesPage.fileAccessRequest}" value="#{version.termsOfUseAndAccess.fileAccessRequest}" />                        
                                                 </p:commandLink>
                                             </li>
-                                            <li>
+                                            <li jsf:rendered="#{!settingsWrapper.publicInstall}">
                                                 <p:commandLink  onclick="if (!testFilesSelected2()) return false;"
                                                                 actionListener="#{EditDatafilesPage.restrictFiles(false)}"
                                                                 update="filesTable,:messagePanel" 

From c992f1908ad750584ba503d140a6850b86f1f1a2 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 2 Jun 2022 17:02:11 -0400
Subject: [PATCH 03/25] add 5.11 release notes

---
 doc/release-notes/5.11-release-notes.md       | 185 ++++++++++++++++++
 doc/release-notes/5663-shib-confirm-email.md  |   7 -
 .../7492_muting_notifications.md              |   9 -
 doc/release-notes/8227-verify-email.md        |   1 -
 .../8295-support-postgresql-14.md             |   8 -
 .../8380-counter-processor-update.md          |   1 -
 doc/release-notes/8456-upgrade-primefaces.md  |   4 -
 .../8525-ingest-optional-skip.md              |   1 -
 .../8533_semantic-api-updates.md              |  14 --
 ...icense-info-in-submit-for-review-pop-up.md |   1 -
 .../8595-cvv-field-solr-update.md             |   3 -
 doc/release-notes/8600-duplicate-templates.md |   9 -
 .../8608-bagit-support-validate-checksums.md  |  10 -
 .../8663-update-create-ds-doc.md              |   5 -
 doc/release-notes/ds54-csp.md                 |   1 -
 15 files changed, 185 insertions(+), 74 deletions(-)
 create mode 100644 doc/release-notes/5.11-release-notes.md
 delete mode 100644 doc/release-notes/5663-shib-confirm-email.md
 delete mode 100644 doc/release-notes/7492_muting_notifications.md
 delete mode 100644 doc/release-notes/8227-verify-email.md
 delete mode 100644 doc/release-notes/8295-support-postgresql-14.md
 delete mode 100644 doc/release-notes/8380-counter-processor-update.md
 delete mode 100644 doc/release-notes/8456-upgrade-primefaces.md
 delete mode 100644 doc/release-notes/8525-ingest-optional-skip.md
 delete mode 100644 doc/release-notes/8533_semantic-api-updates.md
 delete mode 100644 doc/release-notes/8561-license-info-in-submit-for-review-pop-up.md
 delete mode 100644 doc/release-notes/8595-cvv-field-solr-update.md
 delete mode 100644 doc/release-notes/8600-duplicate-templates.md
 delete mode 100644 doc/release-notes/8608-bagit-support-validate-checksums.md
 delete mode 100644 doc/release-notes/8663-update-create-ds-doc.md
 delete mode 100644 doc/release-notes/ds54-csp.md

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
new file mode 100644
index 00000000000..8bcceac6b08
--- /dev/null
+++ b/doc/release-notes/5.11-release-notes.md
@@ -0,0 +1,185 @@
+# Dataverse Software 5.11
+
+This release brings new features, enhancements, and bug fixes to the Dataverse Software. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
+
+## Release Highlights
+
+### Terms of Access or Request Access Required for Restricted Files
+
+Beginning in this release, Restricted Files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled. See #8191.
+
+In the "Notes for Dataverse Installation Administrators" section, we have provided a query to help proactively identify datasets that need to be updated.
+
+### Muting Notifications
+
+Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). See #7492.
+
+## Major Use Cases and Infrastructure Enhancements
+
+Changes and fixes in this release include:
+
+- Terms of Access or Request Access required for restricted files. (Issue #8191, PR #8308)
+- Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). (Issue #7492, PR #8530)
+- Tabular ingest can be skipped via API. (Issue #8525, PR #8532)
+- The "Verify Email" button has been changed to "Send Verification Email" and rather than sometimes showing a popup now always sends a fresh verification email (and invalidates previous verification emails). (Issue #8227, PR #8579)
+- For Shibboleth users, the `emailconfirmed` timestamp is now set on login for Shibboleth users and the UI should show "Verified". (Issue #5663, PR #8579)
+- Information about the license selection (or custom terms) is now available in the confirmation popup when contributors click "Submit for Review". Previously, this was only available in the confirmation popup for the "Publish" button, which contributors do not see. (Issue #8561, PR #8691)
+- For installations configured to support multiple languages, controlled vocabulary fields that do not allow multiple entries (e.g. journalArticleType) are now indexed properly. (Issue #8595, PR #8601, PR #8624)
+- Harvesting now works when the Dublin core "language" field is set is set #8139. (Issue #8139, PR #8689)
+- The API endpoint for listing notifications has been enhanced to show the subject, text, and timestamp of notifications. (Issue #8487, PR #8530)
+- The API Guide has been updated to explain that the `Content-type` header is now (as of Dataverse 5.6) necessary to create datasets via native API. (Issue #8663, PR #8676)
+- Admin API endpoints have been added to find and delete dataset templates. (Issue 8600, PR #8706)
+- The BagIt file handler detects and transforms zip files with a BagIt package format into Dataverse data files, validating checksums along the way. See the [BagIt File Handler](https://guides.dataverse.org/en/5.11/installation//config.html#bagit-file-handler) section of the Installation Guide for details. (Issue #8608, PR #8677)
+- PostgreSQL 14 can now be used (though we've tested mostly with 13). PostgreSQL 10+ is required. (Issue #8295, PR #8296)
+- As always, widgets can be embedded in the `<iframe>` HTML tag, but the HTTP header "Content-Security-Policy" is now being sent on non-widget pages to prevent them from being embedded. (PR #8662)
+- URIs in the the experimental Semantic API has changed (details below). (Issue #8533, PR #8592)
+- Installations running Make Data Count can upgrade to Counter Processor-0.1.04. (Issue #8380, PR #8391)
+- PrimeFaces, the UI framework we use, has been upgraded from 10 to 11. (Issue #8456, PR #8652)
+
+## Notes for Dataverse Installation Administrators
+
+### Identifying Datasets Requiring Terms of Access or Request Access Changes
+
+In support of the change to require either Terms of Access or Request Access for all Restricted Files, we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access or Request Access enabled:
+
+https://github.com/IQSS/dataverse/blob/v5.11/scripts/issues/8191/datasets_without_toa_or_request_access
+
+This will allow you to reach out to those dataset owners as appropriate.
+
+### PostgreSQL Version 10+ Required
+
+This release upgrades the bundled PostgreSQL JDBC driver to support major version 14.
+
+Note that the newer PostgreSQL driver required a Flyway version bump, which entails positive and negative consequences:
+
+- The newer version of Flyway supports PostgreSQL 14 and includes a number of security fixes.
+- As of version 8.0 the Flyway Community Edition dropped support for PostgreSQL 9.6 and older.
+
+This means that as foreshadowed in the 5.10 and 5.10.1 release notes, version 10 or higher of PostgreSQL is now required. For suggested upgrade steps, please see "PostgreSQL Update" in the release notes for 5.10: https://github.com/IQSS/dataverse/releases/tag/v5.10
+
+### Counter Processor 0.1.04 Support
+
+This release includes support for counter-processor-0.1.04 for processing Make Data Count metrics. If you are running Make Data Counts support, you should reinstall/reconfigure counter-processor as described in the latest Guides. (For existing installations, note that counter-processor-0.1.04 requires a newer version of Python so you will need to follow the full counter-processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse.)
+
+## New JVM Options and DB Settings
+
+The following DB settings have been added:
+
+- `:ShowMuteOptions`
+- `:AlwaysMuted`
+- `:NeverMuted`
+- `:CreateDataFilesMaxErrorsToDisplay`
+- `:BagItHandlerEnabled`
+- `:BagValidatorJobPoolSize`
+- `:BagValidatorMaxErrors`
+- `:BagValidatorJobWaitInterval`
+
+See the [Database Settings](https://guides.dataverse.org/en/5.11/installation/config.html#database-settings) section of the Guides for more information.
+
+## Notes for Developers and Integrators
+
+In the "Backward Incompatibilities" section below, note changes in the Semantic Metadata API.
+
+### Semantic API Changes
+
+This release include an update to the experimental semantic API and the underlying assignment of URIs to metadata block terms that are not explicitly mapped to terms in community vocabularies. The change affects the output of the OAI_ORE metadata export, the OAI_ORE file in archival bags, and the input/output allowed for those terms in the semantic API.
+
+For those updating integrating code or existing files intended for input into this release of Dataverse, URIs of the form...
+
+`https://dataverse.org/schema/<block name>/<parentField name>#<childField title>`
+
+and
+
+`https://dataverse.org/schema/<block name>/<Field title>`
+
+...are both replaced with URIs of the form...
+
+`https://dataverse.org/schema/<block name>/<Field name>`.
+
+## Backward Incompatibilities
+
+### Create Dataset API Requires Content-type Header (Since 5.6)
+
+Due to a code change introduced in Dataverse 5.6, calls to the native API without the `Content-type` header will fail to create a dataset. The API Guide has been updated to indicate the necessity of this header: https://guides.dataverse.org/en/5.11/api/native-api.html#create-a-dataset-in-a-dataverse-collection
+
+## Complete List of Changes
+
+For the complete list of code changes in this release, see the [5.11 Milestone](https://github.com/IQSS/dataverse/milestone/103?closed=1) in Github.
+
+For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
+
+## Installation
+
+If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.11/installation/). Please also contact us to get added to the [Dataverse Project Map](https://guides.dataverse.org/en/5.11/installation/config.html#putting-your-dataverse-installation-on-the-map-at-dataverse-org) if you have not done so already.
+
+## Upgrade Instructions
+
+0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.11.
+
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.  
+
+In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed.
+
+`export PAYARA=/usr/local/payara5`
+
+(or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell)
+
+1\. Undeploy the previous version.
+
+- `$PAYARA/bin/asadmin list-applications`
+- `$PAYARA/bin/asadmin undeploy dataverse<-version>`
+
+2\. Stop Payara and remove the generated directory
+
+- `service payara stop`
+- `rm -rf $PAYARA/glassfish/domains/domain1/generated`
+
+3\. Start Payara
+
+- `service payara start`
+  
+4\. Deploy this version.
+
+- `$PAYARA/bin/asadmin deploy dataverse-5.11.war`
+
+5\. Restart Payara
+
+- `service payara stop`
+- `service payara start`
+
+6\. Reload citation metadata block
+
+   `wget https://github.com/IQSS/dataverse/releases/download/v5.11/citation.tsv`
+   `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"`
+
+7\. Update Solr `schema.xml`
+
+Note that if you have custom metadata blocks you can skip this step and proceed to the next one.
+
+Edit `schema.xml` and for `journalArticleType` change `multiValued` from "false" to "true" and then restart Solr. Alternatively, download and use the version from https://github.com/IQSS/dataverse/releases/download/v5.11/schema.xml . By default the file can be found at `/usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml`.
+
+7b\. For installations with custom metadata blocks
+
+Use the script provided in the release to add the custom fields to the base `schema.xml` installed in the previous step.
+
+```
+   wget https://github.com/IQSS/dataverse/releases/download/v5.11/update-fields.sh
+   chmod +x update-fields.sh
+   curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-8.11.1/server/solr/collection1/conf/schema.xml
+```
+
+(Note that the curl command above calls the admin API on `localhost` to obtain the list of the custom fields. In the unlikely case that you are running the main Dataverse Application and Solr on different servers, generate the `schema.xml` on the application node, then copy it onto the Solr server.)
+
+8\. Re-export metadata files (only OAI_ORE is affected)
+
+People archiving Bags should re-archive. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/5.11/admin/metadataexport.html#batch-exports-through-the-api)
+
+9\. (Optional) Delete duplicate templates in database
+
+Prior to this release making a copy of a dataset template was creating two copies, only one of which is visible in the dataverse collection and usable. The other was not being assigned a collection was invisible to the user (#8600).
+
+If you would like to remove these orphan templates you may run the following script:
+
+https://github.com/IQSS/dataverse/blob/v5.11/scripts/issues/8600/delete_orphan_templates_8600.sh
+
+Also, admin APIs for finding and deleting templates have been added: https://guides.dataverse.org/en/5.11/api/native-api.html#list-dataset-templates
diff --git a/doc/release-notes/5663-shib-confirm-email.md b/doc/release-notes/5663-shib-confirm-email.md
deleted file mode 100644
index b6ef4306d4b..00000000000
--- a/doc/release-notes/5663-shib-confirm-email.md
+++ /dev/null
@@ -1,7 +0,0 @@
-For Shib users we now set the emailconfirmed timestamp on login. (The guides say we do this already but are wrong. It was only being set on account creation.)
-
-For Shib users, I also prevent "check for your welcome email to verify your address" from being shown in the in-app welcome/new account notification.
-
-I put in a check to make sure Shib users never get a "verify your email address" email notification.
-
-Finally, I removed the hasNoStaleVerificationTokens check from the hasVerifiedEmail method. We've never worried about if there are stale verification tokens in the database or not and this check was preventing "Verified" from being shown, even when the user has a timestamp (the timestamp being the way we know if an email is verified or not).
diff --git a/doc/release-notes/7492_muting_notifications.md b/doc/release-notes/7492_muting_notifications.md
deleted file mode 100644
index 18c82b3a106..00000000000
--- a/doc/release-notes/7492_muting_notifications.md
+++ /dev/null
@@ -1,9 +0,0 @@
-This release has a new feature that allows users to control which notifications they receive. How to enable this feature and which options can be set by the admins is described in the [Letting users manage receiving notifications section of the Admin Guide](https://guides.dataverse.org/en/latest/admin/user-administration.html#letting-users-manage-receiving-notifications). See also #7492.
-
-In addition, the existing API endpoint for listing notifications has been enhanced to show the subject, text, and timestamp of notifications. See also #8487.
-
-## New DB Settings
-
-- :ShowMuteOptions
-- :AlwaysMuted
-- :NeverMuted
diff --git a/doc/release-notes/8227-verify-email.md b/doc/release-notes/8227-verify-email.md
deleted file mode 100644
index 0f3faa5436a..00000000000
--- a/doc/release-notes/8227-verify-email.md
+++ /dev/null
@@ -1 +0,0 @@
-The "Verify Email" button has been changed to "Send Verification Email" and rather than sometimes showing a popup now always sends a fresh verification email (and invalidates previous verification emails).
diff --git a/doc/release-notes/8295-support-postgresql-14.md b/doc/release-notes/8295-support-postgresql-14.md
deleted file mode 100644
index 7d658a5e65e..00000000000
--- a/doc/release-notes/8295-support-postgresql-14.md
+++ /dev/null
@@ -1,8 +0,0 @@
-This release upgrades the bundled PostgreSQL JDBC driver to support major version 14.
-
-Note that the newer PostgreSQL driver required a Flyway version bump, which entails positive and negative consequences:
-
-- The newer version of Flyway supports PostgreSQL 14 and includes a number of security fixes.
-- As of version 8.0 the Flyway Community Edition dropped support for PostgreSQL 9.6 and older.
-
-Upgrade instructions may be found under “PostgreSQL Update” in the 5.10 release notes: https://github.com/IQSS/dataverse/releases/tag/v5.10
diff --git a/doc/release-notes/8380-counter-processor-update.md b/doc/release-notes/8380-counter-processor-update.md
deleted file mode 100644
index 8c8bd400d85..00000000000
--- a/doc/release-notes/8380-counter-processor-update.md
+++ /dev/null
@@ -1 +0,0 @@
-This release includes support for counter-processor-0.1.04 for processing Make Data Count metrics. If you are running Make Data Counts support, you should reinstall/reconfigure counter-processor as described in the latest Guides. (For existing installations, note that counter-processor-0.1.04 requires a newer version of python so you will need to follow the full counter-processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse.)
diff --git a/doc/release-notes/8456-upgrade-primefaces.md b/doc/release-notes/8456-upgrade-primefaces.md
deleted file mode 100644
index 6c7fad9fec8..00000000000
--- a/doc/release-notes/8456-upgrade-primefaces.md
+++ /dev/null
@@ -1,4 +0,0 @@
-### Primefaces 11
-
-Primefaces, the open source UI framework upon which the Dataverse front end is built, has been updated to the most recent version. This provides security updates and bug fixes and will also allow Dataverse developers to take advantage of new features and enhancements.
-
diff --git a/doc/release-notes/8525-ingest-optional-skip.md b/doc/release-notes/8525-ingest-optional-skip.md
deleted file mode 100644
index dfec1336ea3..00000000000
--- a/doc/release-notes/8525-ingest-optional-skip.md
+++ /dev/null
@@ -1 +0,0 @@
-Tabular ingest can be skipped via API.
diff --git a/doc/release-notes/8533_semantic-api-updates.md b/doc/release-notes/8533_semantic-api-updates.md
deleted file mode 100644
index a0391ca0dc3..00000000000
--- a/doc/release-notes/8533_semantic-api-updates.md
+++ /dev/null
@@ -1,14 +0,0 @@
-
-## Notes for Developers and Integrators
-
-This release include an update to the experimental semantic API and the underlying assignment of URIs to metadatablock termshat are not explicitly mapped to terms in community vocabularies. The change affects the output of the OAI_ORE metadata export, the OAI_ORE file in archival bags, and the input/out allowed for those terms in the semantic api. For those updating integrating code or existing files intended for input into this release of Dataverse: URIs of the form:
-  https://dataverse.org/schema/<block name>/<parentField name>#<childField title>, and
-  https://dataverse.org/schema/<block name>/<Field title>
-  are both replaced with URIs of the form:
-  https://dataverse.org/schema/<block name>/<Field name>
-
-## Additional Release Steps
-
-Upgrade should include re-export of metadata files (only the OAI_ORE is affected).
-
-For this PR and other changes coming from DataCommons, it will also be advisable for people archiving Bags to re-archive. More detail on the overall set if changes in those tbd PRs.
diff --git a/doc/release-notes/8561-license-info-in-submit-for-review-pop-up.md b/doc/release-notes/8561-license-info-in-submit-for-review-pop-up.md
deleted file mode 100644
index c31fd93b907..00000000000
--- a/doc/release-notes/8561-license-info-in-submit-for-review-pop-up.md
+++ /dev/null
@@ -1 +0,0 @@
-Information about the license selection (or custom terms) is now available in the confirmation popup when contributors click "Submit for Review". Previously, this was only available in the confirmation popup for the "Publish" button, which contributors do not see.
diff --git a/doc/release-notes/8595-cvv-field-solr-update.md b/doc/release-notes/8595-cvv-field-solr-update.md
deleted file mode 100644
index 68f6f555497..00000000000
--- a/doc/release-notes/8595-cvv-field-solr-update.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Controlled vocabulary fields that do not allow multiple entries are not indexed properly in Dataverse instances configured to support multiple languages. This release fixes the schema.xml file for the one field affected in the standard metadata blocks (journalArticleType) and updates the api/admin/index/solr/schema to provide the correct information for use with the update-fields.sh script described in the [Metadata Customization section of the Admin Guide](https://guides.dataverse.org/en/latest/admin/metadatacustomization.html#updating-the-solr-schema).
-
-The release should include updating the schema.xml file for solr or running the update-fields.sh script and reindexing (whatever standard instructions we give for schema changes.)
\ No newline at end of file
diff --git a/doc/release-notes/8600-duplicate-templates.md b/doc/release-notes/8600-duplicate-templates.md
deleted file mode 100644
index c490756760d..00000000000
--- a/doc/release-notes/8600-duplicate-templates.md
+++ /dev/null
@@ -1,9 +0,0 @@
-###Duplicate Templates in Database
-Prior to this release making a copy of a Dataset Template was creating two copies, one of which is visible in the dataverse, the other not being assigned 
-a dataverse was invisible to the user (https://github.com/IQSS/dataverse/issues/8600). This release fixes the issue. 
-
-If you would like to remove these orphan templates you may run the follwing script:
-
-https://github.com/IQSS/dataverse/raw/develop/scripts/issues/8600/delete_orphan_templates_8600.sh
-
-Also, admin APIs for finding and deleting templates have been added.
diff --git a/doc/release-notes/8608-bagit-support-validate-checksums.md b/doc/release-notes/8608-bagit-support-validate-checksums.md
deleted file mode 100644
index f071dbe4039..00000000000
--- a/doc/release-notes/8608-bagit-support-validate-checksums.md
+++ /dev/null
@@ -1,10 +0,0 @@
-## BagIt Support - Automatic checksum validation on zip file upload
-The BagIt file handler detects and transforms zip files with a BagIt package format into Dataverse DataFiles. The system validates the checksums of the files in the package payload as described in the first manifest file with a hash algorithm that we support. Take a look at `BagChecksumType class <https://github.com/IQSS/dataverse/tree/develop/src/main/java/edu/harvard/iq/dataverse/util/bagit/BagChecksumType.java>`_ for the list of the currently supported hash algorithms.
-
-The handler will not allow packages with checksum errors. The first 5 errors will be displayed to the user. This is configurable though database settings.
-
-The checksum validation uses a thread pool to improve performance. This thread pool can be adjusted to your Dataverse installation requirements.
-
-The BagIt file handler is disabled by default. Use the ``:BagItHandlerEnabled`` database settings to enable it: ``curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:BagItHandlerEnabled``
-
-For more configuration settings see the user guide: https://guides.dataverse.org/en/latest/installation/config.html#bagit-file-handler
\ No newline at end of file
diff --git a/doc/release-notes/8663-update-create-ds-doc.md b/doc/release-notes/8663-update-create-ds-doc.md
deleted file mode 100644
index 2ca164a465e..00000000000
--- a/doc/release-notes/8663-update-create-ds-doc.md
+++ /dev/null
@@ -1,5 +0,0 @@
-## Release Highlights
-
-### Documentation of Create Dataset API has been Updated
-
-Previous versions of the guides omitted the content header in the example API call for creating a datset. Due to a code change introduced in Dataverse version 5.6 calls without the content header will fail.
diff --git a/doc/release-notes/ds54-csp.md b/doc/release-notes/ds54-csp.md
deleted file mode 100644
index e8ba16f38c7..00000000000
--- a/doc/release-notes/ds54-csp.md
+++ /dev/null
@@ -1 +0,0 @@
-As always, widgets can be embedded in the `<iframe>` HTML tag, but the HTTP header "Content-Security-Policy" is now being sent on non-widget pages to prevent them from being embedded.

From 4dd4d1afe5df32469faf724cbbd01a938d0023eb Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Thu, 2 Jun 2022 17:02:39 -0400
Subject: [PATCH 04/25] deleted: 8191-require-toa-or-request-access.md #8763

---
 .../8191-require-toa-or-request-access.md       | 17 -----------------
 1 file changed, 17 deletions(-)
 delete mode 100644 doc/release-notes/8191-require-toa-or-request-access.md

diff --git a/doc/release-notes/8191-require-toa-or-request-access.md b/doc/release-notes/8191-require-toa-or-request-access.md
deleted file mode 100644
index d78161973d3..00000000000
--- a/doc/release-notes/8191-require-toa-or-request-access.md
+++ /dev/null
@@ -1,17 +0,0 @@
-## Release Highlights
-
-### Terms of Access or Request Access Required for Restricted Files
-
-Beginning in this release, Restricted Files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled.
-
-In the "Notes for Dataverse Installation Administrators" section, we have provided a query to help proactively identify datasets that need to be updated.
-
-## Notes for Dataverse Installation Administrators
-
-### Identifying Datasets Requiring Terms of Access or Request Access Changes
-
-In support of the change to require either Terms of Access or Request Access for all Restricted Files, we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access or Request Access enabled:
-
-https://github.com/IQSS/dataverse/raw/develop/scripts/issues/8191/
-
-This will allow you to reach out to those dataset owners as appropriate.

From 75705262f984bab8b873d263876b63548505871b Mon Sep 17 00:00:00 2001
From: Stephen Kraffmiller <skraffmiller@hmdc.harvard.edu>
Date: Fri, 3 Jun 2022 09:45:15 -0400
Subject: [PATCH 05/25] #165 ensure ordering of MDBs with Citation first

---
 .../harvard/iq/dataverse/DatasetVersionUI.java   | 16 +++++++++-------
 .../edu/harvard/iq/dataverse/MetadataBlock.java  | 10 +++++++++-
 .../command/impl/MoveDataverseCommandTest.java   |  2 ++
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java
index b20cf0debaa..d09457c86bf 100644
--- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java
+++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionUI.java
@@ -18,6 +18,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
+import static java.util.stream.Collectors.toList;
 import javax.ejb.EJB;
 import javax.faces.view.ViewScoped;
 import javax.inject.Named;
@@ -39,22 +41,22 @@ public class DatasetVersionUI implements Serializable {
     public DatasetVersionUI() {
     }
 
-    private Map<MetadataBlock, List<DatasetField>> metadataBlocksForView = new HashMap<>();
-    private Map<MetadataBlock, List<DatasetField>> metadataBlocksForEdit = new HashMap<>();
+    private TreeMap<MetadataBlock, List<DatasetField>> metadataBlocksForView = new TreeMap<>();
+    private TreeMap<MetadataBlock, List<DatasetField>> metadataBlocksForEdit = new TreeMap<>();
 
-    public Map<MetadataBlock, List<DatasetField>> getMetadataBlocksForView() {
+    public TreeMap<MetadataBlock, List<DatasetField>> getMetadataBlocksForView() {
         return metadataBlocksForView;
     }
 
-    public void setMetadataBlocksForView(Map<MetadataBlock, List<DatasetField>> metadataBlocksForView) {
+    public void setMetadataBlocksForView(TreeMap<MetadataBlock, List<DatasetField>> metadataBlocksForView) {
         this.metadataBlocksForView = metadataBlocksForView;
     }
 
-    public Map<MetadataBlock, List<DatasetField>> getMetadataBlocksForEdit() {
+    public TreeMap<MetadataBlock, List<DatasetField>> getMetadataBlocksForEdit() {
         return metadataBlocksForEdit;
     }
 
-    public void setMetadataBlocksForEdit(Map<MetadataBlock, List<DatasetField>> metadataBlocksForEdit) {
+    public void setMetadataBlocksForEdit(TreeMap<MetadataBlock, List<DatasetField>> metadataBlocksForEdit) {
         this.metadataBlocksForEdit = metadataBlocksForEdit;
     }
     
@@ -417,7 +419,7 @@ public void setMetadataValueBlocks(DatasetVersion datasetVersion) {
                     actualMDB.add(mdbTest);
                 }
             }
-        }       
+        }
         
         for (MetadataBlock mdb : actualMDB) {
             mdb.setEmpty(true);
diff --git a/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java b/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java
index 039915c7201..844c0ec5be7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java
+++ b/src/main/java/edu/harvard/iq/dataverse/MetadataBlock.java
@@ -35,7 +35,7 @@
     @NamedQuery( name="MetadataBlock.findByName", query = "SELECT mdb FROM MetadataBlock mdb WHERE mdb.name=:name")
 })
 @Entity
-public class MetadataBlock implements Serializable {
+public class MetadataBlock implements Serializable, Comparable {
 
     private static final long serialVersionUID = 1L;
     
@@ -210,4 +210,12 @@ public String getLocaleDisplayName()
             return displayName;
         }
     }
+
+    @Override
+    public int compareTo(Object arg0) {
+        //guaranteeing that citation will be shown first with custom blocks in order of creation
+        MetadataBlock other = (MetadataBlock) arg0;
+        Long t = "citation".equals(name) ? -1 : this.getId();
+        return t.compareTo("citation".equals(other.name) ? -1 : other.getId());
+    }
 }
diff --git a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommandTest.java b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommandTest.java
index e6f7a5ea34f..13b60f875d5 100644
--- a/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommandTest.java
+++ b/src/test/java/edu/harvard/iq/dataverse/engine/command/impl/MoveDataverseCommandTest.java
@@ -156,9 +156,11 @@ public void setUp() {
         mbA = new MetadataBlock();
         mbA.setOwner(root);
         mbA.setId(1l);
+        mbA.setName("mbA");
         mbB = new MetadataBlock();
         mbB.setOwner(childE);
         mbB.setId(2l);
+        mbB.setName("mbB");
         mbsE.add(mbB);
         mbsEE.add(mbA);
         mbsEE.add(mbB);

From 29e480c3e4c6f57e897ceaf36638e849c31f63f9 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Fri, 3 Jun 2022 16:20:53 -0400
Subject: [PATCH 06/25] add :BagGeneratorThreads feature to list #8763

---
 doc/release-notes/5.11-release-notes.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 8bcceac6b08..b67e9a67f48 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -30,6 +30,7 @@ Changes and fixes in this release include:
 - The API Guide has been updated to explain that the `Content-type` header is now (as of Dataverse 5.6) necessary to create datasets via native API. (Issue #8663, PR #8676)
 - Admin API endpoints have been added to find and delete dataset templates. (Issue 8600, PR #8706)
 - The BagIt file handler detects and transforms zip files with a BagIt package format into Dataverse data files, validating checksums along the way. See the [BagIt File Handler](https://guides.dataverse.org/en/5.11/installation//config.html#bagit-file-handler) section of the Installation Guide for details. (Issue #8608, PR #8677)
+- For BagIt Export, the number of threads used when zipping data files into an archival bag is now configurable using the `:BagGeneratorThreads` database setting. (Issue #8602, PR #8606)
 - PostgreSQL 14 can now be used (though we've tested mostly with 13). PostgreSQL 10+ is required. (Issue #8295, PR #8296)
 - As always, widgets can be embedded in the `<iframe>` HTML tag, but the HTTP header "Content-Security-Policy" is now being sent on non-widget pages to prevent them from being embedded. (PR #8662)
 - URIs in the the experimental Semantic API has changed (details below). (Issue #8533, PR #8592)
@@ -73,6 +74,7 @@ The following DB settings have been added:
 - `:BagValidatorJobPoolSize`
 - `:BagValidatorMaxErrors`
 - `:BagValidatorJobWaitInterval`
+- `:BagGeneratorThreads`
 
 See the [Database Settings](https://guides.dataverse.org/en/5.11/installation/config.html#database-settings) section of the Guides for more information.
 

From 276b356ae64102cb38f15ecdab3a59d786bfb725 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20Haarla=CC=88nder?= <haarlaender@mpdl.mpg.de>
Date: Wed, 8 Jun 2022 14:38:44 +0200
Subject: [PATCH 07/25] IQSS#8726 Adapt integration tests for better HTTP Range
 support

---
 .../edu/harvard/iq/dataverse/api/FilesIT.java | 38 ++++++++-----------
 1 file changed, 15 insertions(+), 23 deletions(-)

diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
index 60cb608a50a..15ee8f1dd82 100644
--- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
+++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java
@@ -24,16 +24,8 @@
 import java.util.ResourceBundle;
 import javax.json.Json;
 import javax.json.JsonObjectBuilder;
-import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
-
-import static javax.ws.rs.core.Response.Status.CREATED;
-import static javax.ws.rs.core.Response.Status.FORBIDDEN;
-import static javax.ws.rs.core.Response.Status.METHOD_NOT_ALLOWED;
-import static javax.ws.rs.core.Response.Status.NOT_FOUND;
-import static javax.ws.rs.core.Response.Status.NO_CONTENT;
-import static javax.ws.rs.core.Response.Status.OK;
-import static javax.ws.rs.core.Response.Status.REQUESTED_RANGE_NOT_SATISFIABLE;
-import static javax.ws.rs.core.Response.Status.UNAUTHORIZED;
+
+import static javax.ws.rs.core.Response.Status.*;
 import static junit.framework.Assert.assertEquals;
 import org.hamcrest.CoreMatchers;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -1634,25 +1626,25 @@ public void testRange() throws IOException {
         // Download the first 10 bytes.
         Response downloadTxtFirst10 = UtilIT.downloadFile(fileIdTxt, "0-9", null, null, authorApiToken);
         downloadTxtFirst10.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("first is t"));
 
         // Download the last 6 bytes.
         Response downloadTxtLast6 = UtilIT.downloadFile(fileIdTxt, "-6", null, null, authorApiToken);
         downloadTxtLast6.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("chest\n"));
 
         // Download some bytes from the middle.
         Response downloadTxtMiddle = UtilIT.downloadFile(fileIdTxt, "09-19", null, null, authorApiToken);
         downloadTxtMiddle.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("the worst\ns"));
 
         // Skip the first 10 bytes and download the rest.
         Response downloadTxtSkipFirst10 = UtilIT.downloadFile(fileIdTxt, "9-", null, null, authorApiToken);
         downloadTxtSkipFirst10.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("the worst\n"
                         + "second is the best\n"
                         + "third is the one with the hairy chest\n"));
@@ -1687,30 +1679,30 @@ public void testRange() throws IOException {
         // first 10 bytes of tabular format
         Response downloadTabFirstTen = UtilIT.downloadFile(fileIdCsv, "0-9", null, null, authorApiToken);
         downloadTabFirstTen.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("name\tpound"));
 
         // first 30 bytes of tabular format
         Response downloadTabFirst30 = UtilIT.downloadFile(fileIdCsv, "0-29", null, null, authorApiToken);
         downloadTabFirst30.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("name\tpounds\tspecies\n"
                         + "\"Marshall\""));
 
         // last 16 bytes of tabular format
         Response downloadTabLast16 = UtilIT.downloadFile(fileIdCsv, "-16", null, null, authorApiToken);
         downloadTabLast16.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("nther\"\t21\t\"cat\"\n"));
 
         Response downloadTabMiddleBytesHeader = UtilIT.downloadFile(fileIdCsv, "1-7", null, null, authorApiToken);
         downloadTabMiddleBytesHeader.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("ame\tpou"));
 
         Response downloadTabMiddleBytesBody = UtilIT.downloadFile(fileIdCsv, "31-43", null, null, authorApiToken);
         downloadTabMiddleBytesBody.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("40\t\"dog\"\n"
                         + "\"Tig"));
 
@@ -1726,19 +1718,19 @@ public void testRange() throws IOException {
         // first ten bytes
         Response downloadOrigFirstTen = UtilIT.downloadFile(fileIdCsv, "0-9", "original", null, authorApiToken);
         downloadOrigFirstTen.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("name,pound"));
 
         // last ten bytes
         Response downloadOrigLastTen = UtilIT.downloadFile(fileIdCsv, "-10", "original", null, authorApiToken);
         downloadOrigLastTen.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("er,21,cat\n"));
 
         // middle bytes
         Response downloadOrigMiddle = UtilIT.downloadFile(fileIdCsv, "29-39", "original", null, authorApiToken);
         downloadOrigMiddle.then().assertThat()
-                .statusCode(OK.getStatusCode())
+                .statusCode(PARTIAL_CONTENT.getStatusCode())
                 .body(equalTo("40,dog\nTige"));
 
         String pathToZipWithImage = "scripts/search/data/binary/trees.zip";
@@ -1760,7 +1752,7 @@ public void testRange() throws IOException {
         String imageThumbPixels = "true";
         Response downloadThumbnail = UtilIT.downloadFile(fileIdPng, "0-149", null, imageThumbPixels, authorApiToken);
 //        downloadThumbnail.prettyPrint();
-        downloadThumbnail.then().assertThat().statusCode(OK.getStatusCode());
+        downloadThumbnail.then().assertThat().statusCode(PARTIAL_CONTENT.getStatusCode());
 
         Response multipleRangesNotSupported = UtilIT.downloadFile(fileIdTxt, "0-9,20-29", null, null, authorApiToken);
         // "Error due to Range header: Only one range is allowed."

From 24181f388ddeee927a2f4a5f90154d5e53d2ba26 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 8 Jun 2022 13:36:42 -0400
Subject: [PATCH 08/25] typo #8763

---
 doc/release-notes/5.11-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index b67e9a67f48..5769fc134f2 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -41,7 +41,7 @@ Changes and fixes in this release include:
 
 ### Identifying Datasets Requiring Terms of Access or Request Access Changes
 
-In support of the change to require either Terms of Access or Request Access for all Restricted Files, we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access or Request Access enabled:
+In support of the change to require either Terms of Access or Request Access for all Restricted Files, we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access nor Request Access enabled:
 
 https://github.com/IQSS/dataverse/blob/v5.11/scripts/issues/8191/datasets_without_toa_or_request_access
 

From c302cd626a51471c69af69c3f0bd0e6297787f26 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 8 Jun 2022 14:41:17 -0400
Subject: [PATCH 09/25] explain more about terms of access and request access
 #8763

---
 doc/release-notes/5.11-release-notes.md | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 5769fc134f2..4411fa513c3 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -6,10 +6,18 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S
 
 ### Terms of Access or Request Access Required for Restricted Files
 
-Beginning in this release, Restricted Files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled. See #8191.
+Beginning in this release, Restricted Files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled.
+
+Published datasets are not affected by this change. Datasets that are in draft and that have neither Terms of Access nor Request Access enabled must be updated to select one or the other (or both).  Otherwise, datasets cannot be futher edited or published. Dataset authors will be able to tell if their dataset is affected by the presence of the following message at the top of their dataset (when they are logged in):
+
+"Datasets with restricted files are required to have Request Access enabled or Terms of Access to help people access the data. Please edit the dataset to confirm Request Access or provide Terms of Access to be in compliance with the policy."
+
+At this point, authors should click "Edit Dataset" then "Terms" and then check the box for "Request Access" or fill in "Terms of Access for Restricted Files" (or both). Afterwards, authors will be able to further edit metadata and publish.
 
 In the "Notes for Dataverse Installation Administrators" section, we have provided a query to help proactively identify datasets that need to be updated.
 
+See also Issue #8191 and PR #8308.
+
 ### Muting Notifications
 
 Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). See #7492.
@@ -41,7 +49,7 @@ Changes and fixes in this release include:
 
 ### Identifying Datasets Requiring Terms of Access or Request Access Changes
 
-In support of the change to require either Terms of Access or Request Access for all Restricted Files, we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access nor Request Access enabled:
+In support of the change to require either Terms of Access or Request Access for all Restricted Files (see above for details), we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access nor Request Access enabled:
 
 https://github.com/IQSS/dataverse/blob/v5.11/scripts/issues/8191/datasets_without_toa_or_request_access
 

From 2f7b3842ef730797eae10f2d0bb99e6b668dc13b Mon Sep 17 00:00:00 2001
From: Julian Gautier <jggautier@ucla.edu>
Date: Wed, 8 Jun 2022 15:16:38 -0400
Subject: [PATCH 10/25] Update 5.11-release-notes.md

Small grammar edits
---
 doc/release-notes/5.11-release-notes.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 4411fa513c3..d6393379394 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -30,10 +30,10 @@ Changes and fixes in this release include:
 - Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). (Issue #7492, PR #8530)
 - Tabular ingest can be skipped via API. (Issue #8525, PR #8532)
 - The "Verify Email" button has been changed to "Send Verification Email" and rather than sometimes showing a popup now always sends a fresh verification email (and invalidates previous verification emails). (Issue #8227, PR #8579)
-- For Shibboleth users, the `emailconfirmed` timestamp is now set on login for Shibboleth users and the UI should show "Verified". (Issue #5663, PR #8579)
+- For Shibboleth users, the `emailconfirmed` timestamp is now set on login and the UI should show "Verified". (Issue #5663, PR #8579)
 - Information about the license selection (or custom terms) is now available in the confirmation popup when contributors click "Submit for Review". Previously, this was only available in the confirmation popup for the "Publish" button, which contributors do not see. (Issue #8561, PR #8691)
 - For installations configured to support multiple languages, controlled vocabulary fields that do not allow multiple entries (e.g. journalArticleType) are now indexed properly. (Issue #8595, PR #8601, PR #8624)
-- Harvesting now works when the Dublin core "language" field is set is set #8139. (Issue #8139, PR #8689)
+- Harvesting now works when the Dublin core "language" field is set. (Issue #8139, PR #8689)
 - The API endpoint for listing notifications has been enhanced to show the subject, text, and timestamp of notifications. (Issue #8487, PR #8530)
 - The API Guide has been updated to explain that the `Content-type` header is now (as of Dataverse 5.6) necessary to create datasets via native API. (Issue #8663, PR #8676)
 - Admin API endpoints have been added to find and delete dataset templates. (Issue 8600, PR #8706)
@@ -41,7 +41,7 @@ Changes and fixes in this release include:
 - For BagIt Export, the number of threads used when zipping data files into an archival bag is now configurable using the `:BagGeneratorThreads` database setting. (Issue #8602, PR #8606)
 - PostgreSQL 14 can now be used (though we've tested mostly with 13). PostgreSQL 10+ is required. (Issue #8295, PR #8296)
 - As always, widgets can be embedded in the `<iframe>` HTML tag, but the HTTP header "Content-Security-Policy" is now being sent on non-widget pages to prevent them from being embedded. (PR #8662)
-- URIs in the the experimental Semantic API has changed (details below). (Issue #8533, PR #8592)
+- URIs in the the experimental Semantic API have changed (details below). (Issue #8533, PR #8592)
 - Installations running Make Data Count can upgrade to Counter Processor-0.1.04. (Issue #8380, PR #8391)
 - PrimeFaces, the UI framework we use, has been upgraded from 10 to 11. (Issue #8456, PR #8652)
 
@@ -49,7 +49,7 @@ Changes and fixes in this release include:
 
 ### Identifying Datasets Requiring Terms of Access or Request Access Changes
 
-In support of the change to require either Terms of Access or Request Access for all Restricted Files (see above for details), we have provided a query to identify datasets in your installation where at least one file has neither Terms of Access nor Request Access enabled:
+In support of the change to require either Terms of Access or Request Access for all restricted files (see above for details), we have provided a query to identify datasets in your installation where at least one restricted file has neither Terms of Access nor Request Access enabled:
 
 https://github.com/IQSS/dataverse/blob/v5.11/scripts/issues/8191/datasets_without_toa_or_request_access
 
@@ -92,7 +92,7 @@ In the "Backward Incompatibilities" section below, note changes in the Semantic
 
 ### Semantic API Changes
 
-This release include an update to the experimental semantic API and the underlying assignment of URIs to metadata block terms that are not explicitly mapped to terms in community vocabularies. The change affects the output of the OAI_ORE metadata export, the OAI_ORE file in archival bags, and the input/output allowed for those terms in the semantic API.
+This release includes an update to the experimental semantic API and the underlying assignment of URIs to metadata block terms that are not explicitly mapped to terms in community vocabularies. The change affects the output of the OAI_ORE metadata export, the OAI_ORE file in archival bags, and the input/output allowed for those terms in the semantic API.
 
 For those updating integrating code or existing files intended for input into this release of Dataverse, URIs of the form...
 

From 85d400f3ec2aaf795274ea7e4ebda9f48139161e Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 8 Jun 2022 15:20:49 -0400
Subject: [PATCH 11/25] use "datasets with restricted files" phrase #8763

---
 doc/release-notes/5.11-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index d6393379394..35791c79474 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -6,7 +6,7 @@ This release brings new features, enhancements, and bug fixes to the Dataverse S
 
 ### Terms of Access or Request Access Required for Restricted Files
 
-Beginning in this release, Restricted Files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled.
+Beginning in this release, datasets with restricted files must have either Terms of Access or Request Access enabled. This change is to ensure that for each file in a Dataverse installation there is a clear path to get to the data, either through requesting access to the data or to provide context about why requesting access is not enabled.
 
 Published datasets are not affected by this change. Datasets that are in draft and that have neither Terms of Access nor Request Access enabled must be updated to select one or the other (or both).  Otherwise, datasets cannot be futher edited or published. Dataset authors will be able to tell if their dataset is affected by the presence of the following message at the top of their dataset (when they are logged in):
 

From b9f01fc59a76b04672c88dc5e14c2974864232a1 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Wed, 8 Jun 2022 15:39:19 -0400
Subject: [PATCH 12/25] put semantic API changes under backward incompat #8763

---
 doc/release-notes/5.11-release-notes.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 35791c79474..384a4c90f44 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -88,7 +88,9 @@ See the [Database Settings](https://guides.dataverse.org/en/5.11/installation/co
 
 ## Notes for Developers and Integrators
 
-In the "Backward Incompatibilities" section below, note changes in the Semantic Metadata API.
+See the "Backward Incompatibilities" section below.
+
+## Backward Incompatibilities
 
 ### Semantic API Changes
 
@@ -106,8 +108,6 @@ and
 
 `https://dataverse.org/schema/<block name>/<Field name>`.
 
-## Backward Incompatibilities
-
 ### Create Dataset API Requires Content-type Header (Since 5.6)
 
 Due to a code change introduced in Dataverse 5.6, calls to the native API without the `Content-type` header will fail to create a dataset. The API Guide has been updated to indicate the necessity of this header: https://guides.dataverse.org/en/5.11/api/native-api.html#create-a-dataset-in-a-dataverse-collection

From a10ca588402ba3422324f8c6121e16b9c0b96281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20Haarla=CC=88nder?= <haarlaender@mpdl.mpg.de>
Date: Thu, 9 Jun 2022 09:54:14 +0200
Subject: [PATCH 13/25] IQSS/dataverse#8726 Release notes for HTTP range
 requests

---
 .../8727-better-http-range-request-support.md        | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 doc/release-notes/8727-better-http-range-request-support.md

diff --git a/doc/release-notes/8727-better-http-range-request-support.md b/doc/release-notes/8727-better-http-range-request-support.md
new file mode 100644
index 00000000000..dc01ac57dfc
--- /dev/null
+++ b/doc/release-notes/8727-better-http-range-request-support.md
@@ -0,0 +1,12 @@
+### HTTP Range Requests: New HTTP status codes and headers for Datafile Access API
+
+The Basic File Access resource for datafiles (/api/access/datafile/$id) was slightly modified in order to comply better with the HTTP specification for range requests.
+
+If the request contains a "Range" header:
+* The returned HTTP status is now 206 (Partial Content) instead of 200
+* A "Content-Range" header is returned containing information about the returned bytes
+* An "Accept-Ranges" header with value "bytes" is returned
+
+CORS rules/headers were modified accordingly:
+* The "Range" header is added to "Access-Control-Allow-Headers"
+* The "Content-Range" and "Accept-Ranges" header are added to "Access-Control-Expose-Headers"

From 03448a5456a16a0d0c210ffd0db8b2609040e2bf Mon Sep 17 00:00:00 2001
From: landreev <leonid@hmdc.harvard.edu>
Date: Thu, 9 Jun 2022 12:04:22 -0400
Subject: [PATCH 14/25] quick change to the language codes line

---
 doc/release-notes/5.11-release-notes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 384a4c90f44..9155b3dbfa3 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -33,7 +33,7 @@ Changes and fixes in this release include:
 - For Shibboleth users, the `emailconfirmed` timestamp is now set on login and the UI should show "Verified". (Issue #5663, PR #8579)
 - Information about the license selection (or custom terms) is now available in the confirmation popup when contributors click "Submit for Review". Previously, this was only available in the confirmation popup for the "Publish" button, which contributors do not see. (Issue #8561, PR #8691)
 - For installations configured to support multiple languages, controlled vocabulary fields that do not allow multiple entries (e.g. journalArticleType) are now indexed properly. (Issue #8595, PR #8601, PR #8624)
-- Harvesting now works when the Dublin core "language" field is set. (Issue #8139, PR #8689)
+- Two-letter ISO-639-1 codes for languages are now supported, in metadata imports and harvesting. (Issue #8139, PR #8689)
 - The API endpoint for listing notifications has been enhanced to show the subject, text, and timestamp of notifications. (Issue #8487, PR #8530)
 - The API Guide has been updated to explain that the `Content-type` header is now (as of Dataverse 5.6) necessary to create datasets via native API. (Issue #8663, PR #8676)
 - Admin API endpoints have been added to find and delete dataset templates. (Issue 8600, PR #8706)

From 537ad57f87aab9eca7d5432670bc4747b6f81ce9 Mon Sep 17 00:00:00 2001
From: Gustavo Durand <scolapasta+github@gmail.com>
Date: Thu, 9 Jun 2022 14:09:52 -0400
Subject: [PATCH 15/25] Create V5.10.1.3__8599-legacy-templates.sql

---
 .../db/migration/V5.10.1.3__8599-legacy-templates.sql  | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 src/main/resources/db/migration/V5.10.1.3__8599-legacy-templates.sql

diff --git a/src/main/resources/db/migration/V5.10.1.3__8599-legacy-templates.sql b/src/main/resources/db/migration/V5.10.1.3__8599-legacy-templates.sql
new file mode 100644
index 00000000000..5027c48df59
--- /dev/null
+++ b/src/main/resources/db/migration/V5.10.1.3__8599-legacy-templates.sql
@@ -0,0 +1,10 @@
+-- this script finds legacy templates that do not hava an associated termsofuseandaccess 
+-- and creates / links a termsofuseandaccess to them
+with  _update as 
+(
+update template set termsofuseandaccess_id =  nextval('termsofuseandaccess_id_seq' )
+where termsofuseandaccess_id is null
+returning termsofuseandaccess_id
+)
+insert into termsofuseandaccess (id, fileaccessrequest, license_id) (select termsofuseandaccess_id, false, 1 from _update)
+

From 19b568dffab289fb3ae36937f390d06f2872b6f1 Mon Sep 17 00:00:00 2001
From: Gustavo Durand <scolapasta+github@gmail.com>
Date: Thu, 9 Jun 2022 14:17:51 -0400
Subject: [PATCH 16/25] Create 8599-legacy-templates

---
 doc/release-notes/8599-legacy-templates | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 doc/release-notes/8599-legacy-templates

diff --git a/doc/release-notes/8599-legacy-templates b/doc/release-notes/8599-legacy-templates
new file mode 100644
index 00000000000..7285a613635
--- /dev/null
+++ b/doc/release-notes/8599-legacy-templates
@@ -0,0 +1,5 @@
+Some older legacy templates did not have an associated termsofuseandaccess linked to them. When custom licenses were added, dataverses that these legacy templates as default would not allow the creation of a new dataset (500 error). 
+
+In this release, we run a script that creates a default empty termsofuseandaccess for each of these templates and links them.
+
+Note the termsofuseandaccess that are created this way default to using the license with id=1 (cc0) and the fileaccessrequest to false. 

From 8c34ce68add6a725996bec583ea57c0648193657 Mon Sep 17 00:00:00 2001
From: Gustavo Durand <scolapasta+github@gmail.com>
Date: Fri, 10 Jun 2022 12:34:25 -0400
Subject: [PATCH 17/25] Update 5.11-release-notes.md

---
 doc/release-notes/5.11-release-notes.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 9155b3dbfa3..4b4f015569d 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -18,6 +18,18 @@ In the "Notes for Dataverse Installation Administrators" section, we have provid
 
 See also Issue #8191 and PR #8308.
 
+### Legacy Templates Issue
+
+When custom license functionality was added, dataverses that had older legacy templates as their default template would not allow the creation of a new dataset (500 error).
+
+This occurred because those legacy templates did not have an associated termsofuseandaccess linked to them. 
+
+In this release, we run a script that creates a default empty termsofuseandaccess for each of these templates and links them.
+
+Note the termsofuseandaccess that are created this way default to using the license with id=1 (cc0) and the fileaccessrequest to false. 
+
+See #8599.
+
 ### Muting Notifications
 
 Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). See #7492.

From cf83d554c658ee01b444a122aa64a4157c5d2834 Mon Sep 17 00:00:00 2001
From: Gustavo Durand <scolapasta+github@gmail.com>
Date: Fri, 10 Jun 2022 12:34:42 -0400
Subject: [PATCH 18/25] Delete 8599-legacy-templates

---
 doc/release-notes/8599-legacy-templates | 5 -----
 1 file changed, 5 deletions(-)
 delete mode 100644 doc/release-notes/8599-legacy-templates

diff --git a/doc/release-notes/8599-legacy-templates b/doc/release-notes/8599-legacy-templates
deleted file mode 100644
index 7285a613635..00000000000
--- a/doc/release-notes/8599-legacy-templates
+++ /dev/null
@@ -1,5 +0,0 @@
-Some older legacy templates did not have an associated termsofuseandaccess linked to them. When custom licenses were added, dataverses that these legacy templates as default would not allow the creation of a new dataset (500 error). 
-
-In this release, we run a script that creates a default empty termsofuseandaccess for each of these templates and links them.
-
-Note the termsofuseandaccess that are created this way default to using the license with id=1 (cc0) and the fileaccessrequest to false. 

From 59d36972aa52d1e7eaab9d695213f2364b9b96f0 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 13 Jun 2022 11:32:17 -0400
Subject: [PATCH 19/25] move legacy template issue to "notes for admins" #8763

---
 doc/release-notes/5.11-release-notes.md | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 4b4f015569d..0ba2ca3eed4 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -18,18 +18,6 @@ In the "Notes for Dataverse Installation Administrators" section, we have provid
 
 See also Issue #8191 and PR #8308.
 
-### Legacy Templates Issue
-
-When custom license functionality was added, dataverses that had older legacy templates as their default template would not allow the creation of a new dataset (500 error).
-
-This occurred because those legacy templates did not have an associated termsofuseandaccess linked to them. 
-
-In this release, we run a script that creates a default empty termsofuseandaccess for each of these templates and links them.
-
-Note the termsofuseandaccess that are created this way default to using the license with id=1 (cc0) and the fileaccessrequest to false. 
-
-See #8599.
-
 ### Muting Notifications
 
 Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). See #7492.
@@ -40,6 +28,7 @@ Changes and fixes in this release include:
 
 - Terms of Access or Request Access required for restricted files. (Issue #8191, PR #8308)
 - Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). (Issue #7492, PR #8530)
+- A 500 error was occuring when creating a dataset if a template did not have an associated "termsofuseandaccess". See "Legacy Templates Issue" below for details. (Issue #8599, PR #8789)
 - Tabular ingest can be skipped via API. (Issue #8525, PR #8532)
 - The "Verify Email" button has been changed to "Send Verification Email" and rather than sometimes showing a popup now always sends a fresh verification email (and invalidates previous verification emails). (Issue #8227, PR #8579)
 - For Shibboleth users, the `emailconfirmed` timestamp is now set on login and the UI should show "Verified". (Issue #5663, PR #8579)
@@ -67,6 +56,18 @@ https://github.com/IQSS/dataverse/blob/v5.11/scripts/issues/8191/datasets_withou
 
 This will allow you to reach out to those dataset owners as appropriate.
 
+### Legacy Templates Issue
+
+When custom license functionality was added, dataverses that had older legacy templates as their default template would not allow the creation of a new dataset (500 error).
+
+This occurred because those legacy templates did not have an associated termsofuseandaccess linked to them.
+
+In this release, we run a script that creates a default empty termsofuseandaccess for each of these templates and links them.
+
+Note the termsofuseandaccess that are created this way default to using the license with id=1 (cc0) and the fileaccessrequest to false.
+
+See also Issue #8599 and PR #8789.
+
 ### PostgreSQL Version 10+ Required
 
 This release upgrades the bundled PostgreSQL JDBC driver to support major version 14.

From 47d416ba6902cc19263a8af55e5a1186a3145136 Mon Sep 17 00:00:00 2001
From: Philip Durbin <philip_durbin@harvard.edu>
Date: Mon, 13 Jun 2022 11:35:12 -0400
Subject: [PATCH 20/25] minor tweaks #8763

---
 doc/release-notes/5.11-release-notes.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/release-notes/5.11-release-notes.md b/doc/release-notes/5.11-release-notes.md
index 0ba2ca3eed4..a51bcec2dac 100644
--- a/doc/release-notes/5.11-release-notes.md
+++ b/doc/release-notes/5.11-release-notes.md
@@ -20,7 +20,7 @@ See also Issue #8191 and PR #8308.
 
 ### Muting Notifications
 
-Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). See #7492.
+Users can control which notifications they receive if the system is [configured to allow this](https://guides.dataverse.org/en/5.11/admin/user-administration.html#letting-users-manage-receiving-notifications). See also Issue #7492 and PR #8530.
 
 ## Major Use Cases and Infrastructure Enhancements
 
@@ -127,7 +127,7 @@ Due to a code change introduced in Dataverse 5.6, calls to the native API withou
 
 ## Complete List of Changes
 
-For the complete list of code changes in this release, see the [5.11 Milestone](https://github.com/IQSS/dataverse/milestone/103?closed=1) in Github.
+For the complete list of code changes in this release, see the [5.11 Milestone](https://github.com/IQSS/dataverse/milestone/103?closed=1) in GitHub.
 
 For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
 
@@ -139,7 +139,7 @@ If this is a new installation, please see our [Installation Guide](https://guide
 
 0\. These instructions assume that you've already successfully upgraded from Dataverse Software 4.x to Dataverse Software 5 following the instructions in the [Dataverse Software 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0). After upgrading from the 4.x series to 5.0, you should progress through the other 5.x releases before attempting the upgrade to 5.11.
 
-If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.  
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
 
 In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed.
 
@@ -160,7 +160,7 @@ In the following commands we assume that Payara 5 is installed in `/usr/local/pa
 3\. Start Payara
 
 - `service payara start`
-  
+
 4\. Deploy this version.
 
 - `$PAYARA/bin/asadmin deploy dataverse-5.11.war`

From 4e828ec036a97ef1fe723d108542bcb367f2102c Mon Sep 17 00:00:00 2001
From: Kevin Condon <kcondon@hmdc.harvard.edu>
Date: Mon, 13 Jun 2022 12:03:29 -0400
Subject: [PATCH 21/25] Update conf.py

Update to v5.11
---
 doc/sphinx-guides/source/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py
index d9439621cb9..82914a35544 100755
--- a/doc/sphinx-guides/source/conf.py
+++ b/doc/sphinx-guides/source/conf.py
@@ -66,9 +66,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '5.10.1'
+version = '5.11'
 # The full version, including alpha/beta/rc tags.
-release = '5.10.1'
+release = '5.11'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

From 1279795a502bd4f38a210f23081d57b6bd11ba08 Mon Sep 17 00:00:00 2001
From: Kevin Condon <kcondon@hmdc.harvard.edu>
Date: Mon, 13 Jun 2022 12:04:18 -0400
Subject: [PATCH 22/25] Update versions.rst

Update to v5.11
---
 doc/sphinx-guides/source/versions.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst
index 21b4ac9d6f4..e4518f4ca65 100755
--- a/doc/sphinx-guides/source/versions.rst
+++ b/doc/sphinx-guides/source/versions.rst
@@ -6,8 +6,9 @@ Dataverse Software Documentation Versions
 
 This list provides a way to refer to the documentation for previous versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases <https://github.com/IQSS/dataverse/releases>`__ page in our GitHub repo.
 
-- 5.10.1
+- 5.11
 
+- `5.10.1 </en/5.10.1/>`__
 - `5.10 </en/5.10/>`__
 - `5.9 </en/5.9/>`__
 - `5.8 </en/5.8/>`__

From 8292d8528880b802f33bd7a02f09b303dc5d652d Mon Sep 17 00:00:00 2001
From: Kevin Condon <kcondon@hmdc.harvard.edu>
Date: Mon, 13 Jun 2022 12:37:54 -0400
Subject: [PATCH 23/25] Update pom.xml

Update to v5.11
---
 modules/dataverse-parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml
index e85608dd0c4..c2f536c4d41 100644
--- a/modules/dataverse-parent/pom.xml
+++ b/modules/dataverse-parent/pom.xml
@@ -129,7 +129,7 @@
  
     <properties>
         <!-- This is a special Maven property name, do not change! -->
-        <revision>5.10.1</revision>
+        <revision>5.11</revision>
     
         <target.java.version>11</target.java.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

From c47291c3676c0bbba9700de0dfc433d63e361dbd Mon Sep 17 00:00:00 2001
From: "don.sizemore" <don.sizemore@github.com>
Date: Mon, 13 Jun 2022 14:57:37 -0400
Subject: [PATCH 24/25] #8790 setting default license doesn't require
 edit-license.json

---
 doc/sphinx-guides/source/api/native-api.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 5cf90359001..5112cc62b5e 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -3872,13 +3872,13 @@ Superusers can change whether an existing license is active (usable for new data
   export STATE=true
   curl -X PUT -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID/:active/$STATE
 
-Superusers can set which license is the default specified by the license ``$ID``:
+Superusers may change the default license by specifying the license ``$ID``:
 
 .. code-block:: bash
 
-  curl -X PUT -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN --data-binary @edit-license.json $SERVER_URL/api/licenses/default/$ID
+  curl -X PUT -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/default/$ID
 
-Superusers can delete a license that is not in use by the license ``$ID``:
+Superusers can delete a license, provided it is not in use, by the license ``$ID``:
 
 .. code-block:: bash
 

From fcfcbdb5d87c06ed49903a3483a649c25ca3dbe7 Mon Sep 17 00:00:00 2001
From: Florian Rhiem <f.rhiem@fz-juelich.de>
Date: Wed, 15 Jun 2022 08:34:01 +0200
Subject: [PATCH 25/25] Add SampleDB to the list of integrations

---
 doc/sphinx-guides/source/admin/integrations.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst
index 8d3f53981e0..f6ca34bf3d4 100644
--- a/doc/sphinx-guides/source/admin/integrations.rst
+++ b/doc/sphinx-guides/source/admin/integrations.rst
@@ -70,6 +70,13 @@ Amnesia is a flexible data anonymization tool that transforms relational and tra
 
 For instructions on depositing or loading data from Dataverse installations to Amnesia, visit https://amnesia.openaire.eu/about-documentation.html
 
+SampleDB
+++++++++
+
+SampleDB is a web-based electronic lab notebook (ELN) with a focus on flexible metadata. SampleDB can export this flexible, process-specific metadata to a new Dataset in a Dataverse installation using the EngMeta Process Metadata block.
+
+For instructions on using the Dataverse export, you can visit https://scientific-it-systems.iffgit.fz-juelich.de/SampleDB/administrator_guide/dataverse_export.html
+
 
 Embedding Data on Websites
 --------------------------