diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index 2443ef8b2e0..028f0140cc9 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -21,6 +21,9 @@ jobs: working-directory: src/main/resources/META-INF run: echo -e "dataverse.feature.api-session-auth=true" >> microprofile-config.properties + - name: Set build number + run: scripts/installer/custom-build-number + - name: Build application war run: mvn package diff --git a/doc/release-notes/6.0-release-notes.md b/doc/release-notes/6.0-release-notes.md new file mode 100644 index 00000000000..df916216f5b --- /dev/null +++ b/doc/release-notes/6.0-release-notes.md @@ -0,0 +1,300 @@ +# Dataverse 6.0 + +This is a platform upgrade release. Payara, Solr, and Java have been upgraded. No features have been added to the Dataverse software itself. Only a handful of bugs were fixed. + +Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project! + +## Release Highlights (Major Upgrades, Breaking Changes) + +This release contains major upgrades to core components. Detailed upgrade instructions can be found below. + +### Runtime + +- The required Java version has been increased from version 11 to 17. + - See PR #9764 for details. +- Payara application server has been upgraded to version 6.2023.8. + - This is a required update. + - Please note that Payara Community 5 has reached [end of life](https://www.payara.fish/products/payara-platform-product-lifecycle/) + - See PR #9685 and PR #9795 for details. +- Solr has been upgraded to version 9.3.0. + - See PR #9787 for details. +- PostgreSQL 13 remains the tested and supported version. + - That said, the installer and Flyway have been upgraded to support PostgreSQL 14 and 15. See the [PostgreSQL](https://guides.dataverse.org/en/6.0/installation/prerequisites.html#postgresql) section of the Installation Guide and PR #9877 for details. + +### Development + +- Removal of Vagrant and Docker All In One (docker-aio), deprecated in Dataverse v5.14. See PR #9838 and PR #9685 for details. +- All tests have been migrated to use JUnit 5 exclusively from now on. See PR #9796 for details. + +## Installation + +If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it! + +Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club! + +You are also very welcome to join the [Global Dataverse Community Consortium](https://dataversecommunity.global) (GDCC). + +## Upgrade Instructions + +Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc. + +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 5.14. + +### Upgrade from Java 11 to Java 17 + +Java 17 is now required for Dataverse. Solr can run under Java 11 or Java 17 but the latter is recommended. In preparation for the Java upgrade, stop both Dataverse/Payara and Solr. + +1. Undeploy Dataverse, if deployed, using the unprivileged service account. + + `sudo -u dataverse /usr/local/payara5/bin/asadmin list-applications` + + `sudo -u dataverse /usr/local/payara5/bin/asadmin undeploy dataverse-5.14` + +1. Stop Payara 5. + + `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain` + +1. Stop Solr 8. + + `sudo systemctl stop solr.service` + +1. Install Java 17. + + Assuming you are using RHEL or a derivative such as Rocky Linux: + + `sudo yum install java-17-openjdk` + +1. Set Java 17 as the default. + + Assuming you are using RHEL or a derivative such as Rocky Linux: + + `sudo alternatives --config java` + +1. Test that Java 17 is the default. + + `java -version` + +### Upgrade from Payara 5 to Payara 6 + +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. + +1. Download Payara 6.2023.8. + + `curl -L -O https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip` + +1. Unzip it to /usr/local (or your preferred location). + + `sudo unzip payara-6.2023.8.zip -d /usr/local/` + +1. Change ownership of the unzipped Payara to your "service" user ("dataverse" by default). + + `sudo chown -R dataverse /usr/local/payara6` + +1. Undeploy Dataverse, if deployed, using the unprivileged service account. + + `sudo -u dataverse /usr/local/payara5/bin/asadmin list-applications` + + `sudo -u dataverse /usr/local/payara5/bin/asadmin undeploy dataverse-5.14` + +1. Stop Payara 5, if running. + + `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain` + +1. Copy Dataverse-related lines from Payara 5 to Payara 6 domain.xml. + + `sudo -u dataverse cp /usr/local/payara6/glassfish/domains/domain1/config/domain.xml /usr/local/payara6/glassfish/domains/domain1/config/domain.xml.orig` + + `sudo egrep 'dataverse|doi' /usr/local/payara5/glassfish/domains/domain1/config/domain.xml > lines.txt` + + `sudo vi /usr/local/payara6/glassfish/domains/domain1/config/domain.xml` + + The lines will appear in two sections, examples shown below (but your content will vary). + + Section 1: system properties (under ``) + + ``` + + + + + + ``` + + Note: if you used the Dataverse installer, you won't have a `dataverse.db.password` property. See "Create password aliases" below. + + Section 2: JVM options (under ``, the one under ``, not under ``) + + ``` + -Ddataverse.files.directory=/usr/local/dvn/data + -Ddataverse.files.file.type=file + -Ddataverse.files.file.label=file + -Ddataverse.files.file.directory=/usr/local/dvn/data + -Ddataverse.rserve.host=localhost + -Ddataverse.rserve.port=6311 + -Ddataverse.rserve.user=rserve + -Ddataverse.rserve.password=rserve + -Ddataverse.auth.password-reset-timeout-in-minutes=60 + -Ddataverse.timerServer=true + -Ddataverse.fqdn=dev1.dataverse.org + -Ddataverse.siteUrl=https://dev1.dataverse.org + -Ddataverse.files.storage-driver-id=file + -Ddoi.username=testaccount + -Ddoi.password=notmypassword + -Ddoi.baseurlstring=https://mds.test.datacite.org/ + -Ddoi.dataciterestapiurlstring=https://api.test.datacite.org + ``` + +1. Check the `Xmx` setting in `domain.xml`. + + Under `/usr/local/payara6/glassfish/domains/domain1/config/domain.xml`, check the `Xmx` setting under ``, where you put the JVM options, not the one under ``. Note that there are two such settings, and you want to adjust the one in the stanza with Dataverse options. This sets the JVM heap size; a good rule of thumb is half of your system's total RAM. You may specify the value in MB (`8192m`) or GB (`8g`). + +1. Copy `jhove.conf` and `jhoveConfig.xsd` from Payara 5, edit and change `payara5` to `payara6`. + + `sudo cp /usr/local/payara5/glassfish/domains/domain1/config/jhove* /usr/local/payara6/glassfish/domains/domain1/config/` + + `sudo chown dataverse /usr/local/payara6/glassfish/domains/domain1/config/jhove*` + + `sudo -u dataverse vi /usr/local/payara6/glassfish/domains/domain1/config/jhove.conf` + +1. Copy logos from Payara 5 to Payara 6. + + These logos are for collections (dataverses). + + `sudo -u dataverse cp -r /usr/local/payara5/glassfish/domains/domain1/docroot/logos /usr/local/payara6/glassfish/domains/domain1/docroot` + +1. If you are using Make Data Count (MDC), edit :MDCLogPath. + + Your `:MDCLogPath` database setting might be pointing to a Payara 5 directory such as `/usr/local/payara5/glassfish/domains/domain1/logs`. If so, edit this to be Payara 6. You'll probably want to copy your logs over as well. + +1. Update systemd unit file (or other init system) from `/usr/local/payara5` to `/usr/local/payara6`, if applicable. + +1. Start Payara. + + `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain` + +1. Create a Java mail resource, replacing "localhost" for mailhost with your mail relay server, and replacing "localhost" for fromaddress with the FQDN of your Dataverse server. + + `sudo -u dataverse /usr/local/payara6/bin/asadmin create-javamail-resource --mailhost "localhost" --mailuser "dataversenotify" --fromaddress "do-not-reply@localhost" mail/notifyMailSession` + +1. Create password aliases for your database, rserve and datacite jvm-options, if you're using them. + + `echo "AS_ADMIN_ALIASPASSWORD=yourDBpassword" > /tmp/dataverse.db.password.txt` + + `sudo -u dataverse /usr/local/payara6/bin/asadmin create-password-alias --passwordfile /tmp/dataverse.db.password.txt` + + When you are prompted "Enter the value for the aliasname operand", enter `dataverse.db.password` + + You should see "Command create-password-alias executed successfully." + + You'll want to perform similar commands for `rserve_password_alias` and `doi_password_alias` if you're using Rserve and/or DataCite. + +1. Enable workaround for FISH-7722. + + The following workaround is for https://github.com/payara/Payara/issues/6337 + + `sudo -u dataverse /usr/local/payara6/bin/asadmin create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED` + +1. Create the network listener on port 8009. + + `sudo -u dataverse /usr/local/payara6/bin/asadmin create-network-listener --protocol http-listener-1 --listenerport 8009 --jkenabled true jk-connector` + +1. Deploy the Dataverse 6.0 war file. + + `sudo -u dataverse /usr/local/payara6/bin/asadmin deploy /path/to/dataverse-6.0.war` + +1. Check that you get a version number from Dataverse. + + This is just a sanity check that Dataverse has been deployed properly. + + `curl http://localhost:8080/api/info/version` + +1. Perform one final Payara restart to ensure that timers are initialized properly. + + `sudo -u dataverse /usr/local/payara6/bin/asadmin stop-domain` + + `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain` + +### Upgrade from Solr 8 to 9 + +Solr has been upgraded to Solr 9. You must install Solr fresh and reindex. You cannot use your old `schema.xml` because the format has changed. + +The instructions below are copied from https://guides.dataverse.org/en/6.0/installation/prerequisites.html#installing-solr and tweaked a bit for an upgrade scenario. + +We assume that you already have a user called "solr" (from the instructions above), added during your initial installation of Solr. We also assume that you have already stopped Solr 8 as explained in the instructions above about upgrading Java. + +1. Become the "solr" user and then download and configure Solr. + + `su - solr` + + `cd /usr/local/solr` + + `wget https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz` + + `tar xvzf solr-9.3.0.tgz` + + `cd solr-9.3.0` + + `cp -r server/solr/configsets/_default server/solr/collection1` + +1. Unzip "dvinstall.zip" from this release. Unzip it into /tmp. Then copy the following files into place. + + `cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf` + + `cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf` + +1. A Dataverse installation requires a change to the jetty.xml file that ships with Solr. + + Edit `/usr/local/solr/solr-9.3.0/server/etc/jetty.xml`, increasing `requestHeaderSize` from `8192` to `102400` + +1. Tell Solr to create the core "collection1" on startup. + + `echo "name=collection1" > /usr/local/solr/solr-9.3.0/server/solr/collection1/core.properties` + +1. Update your init script. + + Your init script may be located at `/etc/systemd/system/solr.service`, for example. Update the path to Solr to be `/usr/local/solr/solr-9.3.0`. + +1. Start Solr using your init script and check collection1. + + The collection1 check below should print out fields Dataverse uses like "dsDescription". + + `systemctl start solr.service` + + `curl http://localhost:8983/solr/collection1/schema/fields` + +1. Reindex Solr. + + For details, see https://guides.dataverse.org/en/6.0/admin/solr-search-index.html but here is the reindex command: + + `curl http://localhost:8080/api/admin/index` + +1. If you have custom metadata blocks installed, you must update your Solr `schema.xml` to include your custom fields. + + For details, please see https://guides.dataverse.org/en/6.0/admin/metadatacustomization.html#updating-the-solr-schema + + At a high level you will be copying custom fields from the output of http://localhost:8080/api/admin/index/solr/schema or using a script to automate this. + +## Potential Archiver Incompatibilities with Payara 6 + +The [Google Cloud and DuraCloud archivers](https://guides.dataverse.org/en/5.14/installation/config.html#bagit-export) may not work in Dataverse 6.0. + +This is due to the archivers' dependence on libraries that include classes in `javax.* packages` that are no longer available. If these classes are actually used when the archivers run, the archivers would fail. As these two archivers require additional setup, they have not been tested in 6.0. Community members using these archivers or considering their use are encouraged to test them with 6.0 and report any errors and/or provide fixes for them that can be included in future releases. + +## Bug Fix for Dataset Templates with Custom Terms of Use + +A bug was fixed for the following scenario: + +- Create a template with custom terms. +- Set that template as the default. +- Try to create a dataset. +- A 500 error appears before the form to create dataset is even shown. + +For more details, see issue #9825 and PR #9892 + +## Complete List of Changes + +For the complete list of code changes in this release, see the [6.0 Milestone](https://github.com/IQSS/dataverse/milestone/109?closed=1) in GitHub. + +## Getting Help + +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org. diff --git a/doc/release-notes/8094-java-17.md b/doc/release-notes/8094-java-17.md deleted file mode 100644 index f3c81145465..00000000000 --- a/doc/release-notes/8094-java-17.md +++ /dev/null @@ -1 +0,0 @@ -Java 17 or higher is now required. diff --git a/doc/release-notes/8305-payara6-ee10-v3.md b/doc/release-notes/8305-payara6-ee10-v3.md deleted file mode 100644 index 94369e0211f..00000000000 --- a/doc/release-notes/8305-payara6-ee10-v3.md +++ /dev/null @@ -1,5 +0,0 @@ -Payara has been updated from version 5 to 6. - -Developers, you are encouraged to upgrade to Payara 6 immediately. - -Sysadmins, instructions on how to upgrade production installations will be written as part of https://github.com/IQSS/dataverse/issues/9340 diff --git a/doc/release-notes/9260-solr930.md b/doc/release-notes/9260-solr930.md deleted file mode 100644 index 07824920b3e..00000000000 --- a/doc/release-notes/9260-solr930.md +++ /dev/null @@ -1,63 +0,0 @@ -Solr has been upgraded to Solr 9. You must install Solr fresh and reindex. You cannot use your old schema.xml because the format has changed. - -Specifically, to install Solr fresh you should follow the instructions for new installations, found at https://guides.dataverse.org/en/9260-solr930/installation/prerequisites.html#installing-solr - -These instructions are copied below and tweaked a bit for an upgrade scenario. - -We assume that you already have a user called "solr" (from the instructions above), added during your initial installation of Solr. - -1. Become the "solr" user and then download and configure Solr. - - ``` - su - solr - cd /usr/local/solr - wget https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz - tar xvzf solr-9.3.0.tgz - cd solr-9.3.0 - cp -r server/solr/configsets/_default server/solr/collection1 - ``` - -1. Unzip "dvinstall.zip" from this release. Unzip it into /tmp. Then copy the following files into place. - - ``` - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf - - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf - ``` - -1. A Dataverse installation requires a change to the jetty.xml file that ships with Solr. - - Edit /usr/local/solr/solr-9.3.0/server/etc/jetty.xml , increasing `requestHeaderSize` from `8192` to `102400` - -1. Tell Solr to create the core "collection1" on startup. - - ``` - echo "name=collection1" > /usr/local/solr/solr-9.3.0/server/solr/collection1/core.properties - ``` - -1. Update your init script. - - Your init script may be located at `/etc/systemd/system/solr.service`, for example. Update the path to Solr to be `/usr/local/solr/solr-9.3.0`. - -1. Start Solr using your init script and check collection1. - - The collection1 check below should print out fields Dataverse uses like "dsDescription". - - ``` - systemctl start solr.service - curl http://localhost:8983/solr/collection1/schema/fields - ``` - -1. Reindex Solr. - - For details, see https://guides.dataverse.org/en/9260-solr930/admin/solr-search-index.html - - ``` - curl http://localhost:8080/api/admin/index - ``` - -1. If you have custom metadata blocks installed, you must update your Solr schema.xml to include your custom fields. - - For details, please see https://guides.dataverse.org/en/9260-solr930/admin/metadatacustomization.html#updating-the-solr-schema - - At a high level you will be copying custom fields from the output of http://localhost:8080/api/admin/index/solr/schema or using a script to automate this. diff --git a/doc/release-notes/9340-payara5to6.md b/doc/release-notes/9340-payara5to6.md deleted file mode 100644 index 68162ef7598..00000000000 --- a/doc/release-notes/9340-payara5to6.md +++ /dev/null @@ -1,132 +0,0 @@ -## Upgrade from Payara 5 to Payara 6 - -1. Download Payara 6.2023.8 as of this writing: - - `curl -L -O https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip` - -1. Unzip it to /usr/local (or your preferred location): - - `sudo unzip payara-6.2023.8.zip -d /usr/local/` - -1. Change ownership of the unzipped Payara to your "service" user ("dataverse" by default): - - `sudo chown -R dataverse /usr/local/payara6` - -1. Undeploy Dataverse (if deployed, using the unprivileged service account. Version 5.14 is assumed in the example below): - - `sudo -u dataverse /usr/local/payara5/bin/asadmin list-applications` - - `sudo -u dataverse /usr/local/payara5/bin/asadmin undeploy dataverse-5.14` - -1. Stop Payara 5: - - `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain` - -1. Copy Dataverse-related lines from Payara 5 to Payara 6 domain.xml: - - `sudo -u dataverse cp /usr/local/payara6/glassfish/domains/domain1/config/domain.xml /usr/local/payara6/glassfish/domains/domain1/config/domain.xml.orig` - - `sudo egrep 'dataverse|doi' /usr/local/payara5/glassfish/domains/domain1/config/domain.xml > lines.txt` - - `sudo vi /usr/local/payara6/glassfish/domains/domain1/config/domain.xml` - - The lines will appear in two sections, examples shown below (but your content will vary). - - Section 1: system properties (under ``) - - ``` - - - - - - ``` - - Note: if you used the Dataverse installer, you won't have a `dataverse.db.password` property. See "Create password aliases" below. - - Section 2: JVM options (under ``, the one under ``, not under ``) - - ``` - -Ddataverse.files.directory=/usr/local/dvn/data - -Ddataverse.files.file.type=file - -Ddataverse.files.file.label=file - -Ddataverse.files.file.directory=/usr/local/dvn/data - -Ddataverse.rserve.host=localhost - -Ddataverse.rserve.port=6311 - -Ddataverse.rserve.user=rserve - -Ddataverse.rserve.password=rserve - -Ddataverse.auth.password-reset-timeout-in-minutes=60 - -Ddataverse.timerServer=true - -Ddataverse.fqdn=dev1.dataverse.org - -Ddataverse.siteUrl=https://dev1.dataverse.org - -Ddataverse.files.storage-driver-id=file - -Ddoi.username=testaccount - -Ddoi.password=notmypassword - -Ddoi.baseurlstring=https://mds.test.datacite.org/ - -Ddoi.dataciterestapiurlstring=https://api.test.datacite.org - ``` - -1. Check the `Xmx` setting in `/usr/local/payara6/glassfish/domains/domain1/config/domain.xml`. (The one under ``, where you put the JVM options, not the one under ``.) Note that there are two such settings, and you want to adjust the one in the stanza with Dataverse options. This sets the JVM heap size; a good rule of thumb is half of your system's total RAM. You may specify the value in MB (`8192m`) or GB (`8g`). - -1. Copy jhove.conf and jhoveConfig.xsd from Payara 5, edit and change payara5 to payara6 - - `sudo cp /usr/local/payara5/glassfish/domains/domain1/config/jhove* /usr/local/payara6/glassfish/domains/domain1/config/` - - `sudo chown dataverse /usr/local/payara6/glassfish/domains/domain1/config/jhove*` - - `sudo -u dataverse vi /usr/local/payara6/glassfish/domains/domain1/config/jhove.conf` - -1. Copy logos from Payara 5 to Payara 6 - - These logos are for collections (dataverses). - - `sudo -u dataverse cp -r /usr/local/payara5/glassfish/domains/domain1/docroot/logos /usr/local/payara6/glassfish/domains/domain1/docroot` - -1. If you are using Make Data Count (MDC), edit :MDCLogPath - - Your `:MDCLogPath` database setting might be pointing to a Payara 5 directory such as `/usr/local/payara5/glassfish/domains/domain1/logs`. If so, edit this to be Payara 6. You'll probably want to copy your logs over as well. - -1. Update systemd unit file (or other init system) from `/usr/local/payara5` to `/usr/local/payara6`, if applicable. - -1. Start Payara: - - `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain` - -1. Create a Java mail resource, replacing "localhost" for mailhost with your mail relay server, and replacing "localhost" for fromaddress with the FQDN of your Dataverse server: - - `sudo -u dataverse /usr/local/payara6/bin/asadmin create-javamail-resource --mailhost "localhost" --mailuser "dataversenotify" --fromaddress "do-not-reply@localhost" mail/notifyMailSession` - -1. Create password aliases for your database, rserve and datacite jvm-options, if you're using them: - - ``` - $ echo "AS_ADMIN_ALIASPASSWORD=yourDBpassword" > /tmp/dataverse.db.password.txt - $ sudo -u dataverse /usr/local/payara6/bin/asadmin create-password-alias --passwordfile /tmp/dataverse.db.password.txt - Enter the value for the aliasname operand> dataverse.db.password - Command create-password-alias executed successfully. - ``` - - You'll want to perform similar commands for `rserve_password_alias` and `doi_password_alias` if you're using Rserve and/or Datacite. - -1. Enable workaround for FISH-7722: - - The following workaround is for https://github.com/payara/Payara/issues/6337 - - `sudo -u dataverse /usr/local/payara6/bin/asadmin create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED` - -1. Create the network listener on port 8009 - - `sudo -u dataverse /usr/local/payara6/bin/asadmin create-network-listener --protocol http-listener-1 --listenerport 8009 --jkenabled true jk-connector` - -1. Deploy the Dataverse 6.0 warfile: - - `sudo -u dataverse /usr/local/payara6/bin/asadmin deploy /path/to/dataverse-6.0.war` - -1. Check that you get a version number from Dataverse: - - `curl http://localhost:8080/api/info/version` - -1. Perform one final Payara restart to ensure that timers are initialized properly: - - `sudo -u dataverse /usr/local/payara6/bin/asadmin stop-domain` - - `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain` diff --git a/doc/release-notes/9782-juni5-transition.md b/doc/release-notes/9782-juni5-transition.md deleted file mode 100644 index b7ffcc0de0d..00000000000 --- a/doc/release-notes/9782-juni5-transition.md +++ /dev/null @@ -1,7 +0,0 @@ -# Migrating all test to JUnit 5 -With this release, we transition all of our test cases (see `src/test/`) to use JUnit 5 only. -Moving forward from JUnit 4 will allow writing tests in more concise and easier ways. -The tests themselves have not been altered, but updated to match JUnit 5 ways. -They have not been extended or dropped coverage; this is mostly a preparation of things to come in the future. -If you are writing tests in JUnit 4 in your feature branches, you need to migrate. -The development guides section of testing has been updated as well. diff --git a/doc/release-notes/9812-archiver-warnings.md b/doc/release-notes/9812-archiver-warnings.md deleted file mode 100644 index 716223b3f46..00000000000 --- a/doc/release-notes/9812-archiver-warnings.md +++ /dev/null @@ -1,7 +0,0 @@ -# Potential Archiver Incompatibilities with Payara6 -The Google Cloud and DuraCloud Archivers (see https://guides.dataverse.org/en/latest/installation/config.html#bagit-export) may not work in v6.0. -This is due to their dependence on libraries that include classes in javax.* packages that are no longer available. -If these classes are actually used when the archivers run, the archivers would fail. -As these two archivers require additional setup, they have not been tested in v6.0. -Community members using these archivers or considering their use are encouraged to test them with v6.0 and report any errors and/or provide fixes for them that can be included in future releases. - diff --git a/doc/release-notes/9838-rm-vagrant.md b/doc/release-notes/9838-rm-vagrant.md deleted file mode 100644 index 910f2e0b2f0..00000000000 --- a/doc/release-notes/9838-rm-vagrant.md +++ /dev/null @@ -1 +0,0 @@ -Vagrant has been removed. See #9838. diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 777d3925f09..0660ec3b071 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -66,9 +66,9 @@ # built documents. # # The short X.Y version. -version = '5.14' +version = '6.0' # The full version, including alpha/beta/rc tags. -release = '5.14' +release = '6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/developers/dependencies.rst b/doc/sphinx-guides/source/developers/dependencies.rst index 65edfa3ffac..0208c49f90a 100644 --- a/doc/sphinx-guides/source/developers/dependencies.rst +++ b/doc/sphinx-guides/source/developers/dependencies.rst @@ -344,8 +344,7 @@ Repositories ------------ Maven receives all dependencies from *repositories*. These can be public like `Maven Central `_ -and others, but you can also use a private repository on premises or in the cloud. Last but not least, you can use -local repositories, which can live next to your application code (see ``local_lib`` dir within the Dataverse Software codebase). +and others, but you can also use a private repository on premises or in the cloud. Repositories are defined within the Dataverse Software POM like this: @@ -364,11 +363,6 @@ Repositories are defined within the Dataverse Software POM like this: http://repository.primefaces.org default - - dvn.private - Local repository for hosting jars not available from network repositories. - file://${project.basedir}/local_lib - You can also add repositories to your local Maven settings, see `docs `_. diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index c77ddc13519..3ac9e955ea2 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -27,6 +27,7 @@ Developer Guide deployment containers making-releases + making-library-releases metadataexport tools unf/index diff --git a/doc/sphinx-guides/source/developers/making-library-releases.rst b/doc/sphinx-guides/source/developers/making-library-releases.rst new file mode 100755 index 00000000000..63b6eeb1c2a --- /dev/null +++ b/doc/sphinx-guides/source/developers/making-library-releases.rst @@ -0,0 +1,93 @@ +======================= +Making Library Releases +======================= + +.. contents:: |toctitle| + :local: + +Introduction +------------ + +Note: See :doc:`making-releases` for Dataverse itself. + +We release Java libraries to Maven Central that are used by Dataverse (and perhaps `other `_ `software `_!): + +- https://central.sonatype.com/namespace/org.dataverse +- https://central.sonatype.com/namespace/io.gdcc + +We release JavaScript/TypeScript libraries to npm: + +- https://www.npmjs.com/package/@iqss/dataverse-design-system + +Maven Central (Java) +-------------------- + +From the perspective of the Maven Central, we are both `producers `_ because we publish/release libraries there and `consumers `_ because we pull down those libraries (and many others) when we build Dataverse. + +Releasing Existing Libraries to Maven Central +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you need to release an existing library, all the setup should be done already. The steps below assume that GitHub Actions are in place to do the heavy lifting for you, such as signing artifacts with GPG. + +Releasing a Snapshot Version to Maven Central +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +`Snapshot `_ releases are published automatically through GitHub Actions (e.g. through a `snapshot workflow `_ for the SWORD library) every time a pull request is merged (or the default branch, typically ``main``, is otherwise updated). + +That is to say, to make a snapshot release, you only need to get one or more commits into the default branch. + +Releasing a Release (Non-Snapshot) Version to Maven Central +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +From a pom.xml it may not be apparent that snapshots like ``6.0-SNAPSHOT`` might be changing under your feet. Browsing the snapshot repository (e.g. our `UNF 6.0-SNAPSHOT `_), may reveal versions changing over time. To finalize the code and stop it from changing, we publish/release what Maven calls a "`release version `_". This will remove ``-SNAPSHOT`` from the version (through an ``mvn`` command). + +Non-snapshot releases (`release `_ versions) are published automatically through GitHub Actions (e.g. through a `release workflow `_), kicked off locally by an ``mvn`` command that invokes the `Maven Release Plugin `_. + +First, run a clean: + +``mvn release:clean`` + +Then run a prepare: + +``mvn release:prepare`` + +The prepare step is interactive. You will be prompted for the following information: + +- the release version (e.g. `2.0.0 `_) +- the git tag to create and push (e.g. `sword2-server-2.0.0 `_) +- the next development (snapshot) version (e.g. `2.0.1-SNAPSHOT `_) + +These examples from the SWORD library. Below is what to expect from the interactive session. In many cases, you can just hit enter to accept the defaults. + +.. code-block:: bash + + [INFO] 5/17 prepare:map-release-versions + What is the release version for "SWORD v2 Common Server Library (forked)"? (sword2-server) 2.0.0: : + [INFO] 6/17 prepare:input-variables + What is the SCM release tag or label for "SWORD v2 Common Server Library (forked)"? (sword2-server) sword2-server-2.0.0: : + [INFO] 7/17 prepare:map-development-versions + What is the new development version for "SWORD v2 Common Server Library (forked)"? (sword2-server) 2.0.1-SNAPSHOT: : + [INFO] 8/17 prepare:rewrite-poms-for-release + +It can take some time for the jar to be visible on Maven Central. You can start by looking on the repo1 server, like this: https://repo1.maven.org/maven2/io/gdcc/sword2-server/2.0.0/ + +Don't bother putting the new version in a pom.xml until you see it on repo1. + +Note that the next snapshot release should be available as well, like this: https://s01.oss.sonatype.org/content/groups/staging/io/gdcc/sword2-server/2.0.1-SNAPSHOT/ + +Releasing a New Library to Maven Central +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +At a high level: + +- Use an existing pom.xml as a starting point. +- Use existing GitHub Actions workflows as a starting point. +- Create secrets in the new library's GitHub repo used by the workflow. +- If you need an entire new namespace, look at previous issues such as https://issues.sonatype.org/browse/OSSRH-94575 and https://issues.sonatype.org/browse/OSSRH-94577 + +npm (JavaScript/TypeScript) +--------------------------- + +Currently, publishing `@iqss/dataverse-design-system `_ to npm done manually. We plan to automate this as part of https://github.com/IQSS/dataverse-frontend/issues/140 + +https://www.npmjs.com/package/js-dataverse is the previous 1.0 version of js-dataverse. No 1.x releases are planned. We plan to publish 2.0 (used by the new frontend) as discussed in https://github.com/IQSS/dataverse-frontend/issues/13 \ No newline at end of file diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index a2575bb5f50..23c4773a06e 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -8,6 +8,8 @@ Making Releases Introduction ------------ +Note: See :doc:`making-library-releases` for how to publish our libraries to Maven Central. + See :doc:`version-control` for background on our branching strategy. The steps below describe making both regular releases and hotfix releases. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 8754f968031..caa878d21df 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -97,10 +97,14 @@ Also note that Payara may utilize more than the default number of file descripto PostgreSQL ---------- +PostgreSQL 13 is recommended because it's the version we test against. Version 10 or higher is required because that's what's `supported by Flyway `_, which we use for database migrations. + +You are welcome to experiment with newer versions of PostgreSQL, but please note that as of PostgreSQL 15, permissions have been restricted on the ``public`` schema (`release notes `_, `EDB blog post `_, `Crunchy Data blog post `_). The Dataverse installer has been updated to restore the old permissions, but this may not be a long term solution. + Installing PostgreSQL ===================== -The application has been tested with PostgreSQL versions up to 13 and version 10+ is required. We recommend installing the latest version that is available for your OS distribution. *For example*, to install PostgreSQL 13 under RHEL7/derivative:: +*For example*, to install PostgreSQL 13 under RHEL7/derivative:: # yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm # yum makecache fast diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index d5ffb2acb66..2000a2097f0 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 5.14 +- 6.0 +- `5.14 `__ - `5.13 `__ - `5.12.1 `__ - `5.12 `__ diff --git a/local_lib/org/dataverse/unf/6.0/unf-6.0.jar b/local_lib/org/dataverse/unf/6.0/unf-6.0.jar deleted file mode 100644 index d2738e2dadd..00000000000 Binary files a/local_lib/org/dataverse/unf/6.0/unf-6.0.jar and /dev/null differ diff --git a/local_lib/org/dataverse/unf/6.0/unf-6.0.jar.md5 b/local_lib/org/dataverse/unf/6.0/unf-6.0.jar.md5 deleted file mode 100644 index 04ca3e73ce8..00000000000 --- a/local_lib/org/dataverse/unf/6.0/unf-6.0.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -bd9b84a9ad737a81a2699ab81541a901 diff --git a/local_lib/org/dataverse/unf/6.0/unf-6.0.jar.sha1 b/local_lib/org/dataverse/unf/6.0/unf-6.0.jar.sha1 deleted file mode 100644 index a48cef32570..00000000000 --- a/local_lib/org/dataverse/unf/6.0/unf-6.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4cad279c362e4c5c17a2058dc2c8f2fc97c76bf8 diff --git a/local_lib/org/dataverse/unf/6.0/unf-6.0.pom b/local_lib/org/dataverse/unf/6.0/unf-6.0.pom deleted file mode 100644 index 06f1508723f..00000000000 --- a/local_lib/org/dataverse/unf/6.0/unf-6.0.pom +++ /dev/null @@ -1,8 +0,0 @@ - - - 4.0.0 - org.dataverse - unf - 6.0 - diff --git a/local_lib/org/dataverse/unf/6.0/unf-6.0.pom.md5 b/local_lib/org/dataverse/unf/6.0/unf-6.0.pom.md5 deleted file mode 100644 index 138bc9c95f6..00000000000 --- a/local_lib/org/dataverse/unf/6.0/unf-6.0.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -230c5b1f5ae71bb2fe80ef9e7209f681 diff --git a/local_lib/org/dataverse/unf/6.0/unf-6.0.pom.sha1 b/local_lib/org/dataverse/unf/6.0/unf-6.0.pom.sha1 deleted file mode 100644 index 689e8045418..00000000000 --- a/local_lib/org/dataverse/unf/6.0/unf-6.0.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -286b819f2fc7432a94b5940c6171be1589f66a37 diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 289abdde625..c45d59e4f5f 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -131,7 +131,7 @@ - 5.14 + 6.0 17 UTF-8 @@ -149,7 +149,7 @@ 6.2023.8 - 42.5.1 + 42.6.0 9.3.0 1.12.290 0.177.0 @@ -414,11 +414,6 @@ Unidata All https://artifacts.unidata.ucar.edu/repository/unidata-all/ - - dvn.private - Local repository for hosting jars not available from network repositories. - file://${project.basedir}/local_lib - oss-sonatype diff --git a/pom.xml b/pom.xml index b6aaedf0586..7ba22d2a076 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ war 1.2.18.4 - 8.5.10 + 9.21.2 1.20.1 0.8.7 5.2.1 diff --git a/scripts/installer/install.py b/scripts/installer/install.py index 700c70dbc28..3aedbd8c6ad 100644 --- a/scripts/installer/install.py +++ b/scripts/installer/install.py @@ -380,12 +380,13 @@ print("Can't connect to PostgresQL as the admin user.\n") sys.exit("Is the server running, have you adjusted pg_hba.conf, etc?") - # 3b. get the Postgres version (do we need it still?) + # 3b. get the Postgres version for new permissions model in versions 15+ try: - pg_full_version = conn.server_version - print("PostgresQL version: "+str(pg_full_version)) + pg_full_version = str(conn.server_version) + pg_major_version = pg_full_version[0:2] + print("PostgreSQL version: "+pg_major_version) except: - print("Warning: Couldn't determine PostgresQL version.") + print("Warning: Couldn't determine PostgreSQL version.") conn.close() # 3c. create role: @@ -410,7 +411,9 @@ else: sys.exit("Couldn't create database or database already exists.\n") - conn_cmd = "GRANT ALL PRIVILEGES on DATABASE "+pgDb+" to "+pgUser+";" + # 3e. set permissions: + + conn_cmd = "GRANT CREATE PRIVILEGES on DATABASE "+pgDb+" to "+pgUser+";" try: cur.execute(conn_cmd) except: @@ -418,6 +421,19 @@ cur.close() conn.close() + if int(pg_major_version) >= 15: + conn_cmd = "GRANT ALL ON SCHEMA public TO "+pgUser+";" + print("PostgreSQL 15 or higher detected. Running " + conn_cmd) + try: + cur.execute(conn_cmd) + except: + if force: + print("WARNING: failed to grant permissions on schema public - continuing, since the --force option was specified") + else: + sys.exit("Couldn't grant privileges on schema public to "+pgUser) + cur.close() + conn.close() + print("Database and role created!") if pgOnly: print("postgres-only setup complete.") diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java index 113a6128364..4fb3ffe6c14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JSONLDUtil.java @@ -52,6 +52,7 @@ import edu.harvard.iq.dataverse.DatasetVersion.VersionState; import edu.harvard.iq.dataverse.license.License; import edu.harvard.iq.dataverse.license.LicenseServiceBean; +import jakarta.json.JsonReader; public class JSONLDUtil { @@ -533,13 +534,11 @@ public static JsonObject decontextualizeJsonLD(String jsonLDString) { try (StringReader rdr = new StringReader(jsonLDString)) { // Use JsonLd to expand/compact to localContext - JsonObject jsonld = Json.createReader(rdr).readObject(); - JsonDocument doc = JsonDocument.of(jsonld); - JsonArray array = null; - try { - array = JsonLd.expand(doc).get(); - jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())) - .get(); + try (JsonReader jsonReader = Json.createReader(rdr)) { + JsonObject jsonld = jsonReader.readObject(); + JsonDocument doc = JsonDocument.of(jsonld); + JsonArray array = JsonLd.expand(doc).get(); + jsonld = JsonLd.compact(JsonDocument.of(array), JsonDocument.of(Json.createObjectBuilder().build())).get(); // jsonld = array.getJsonObject(0); logger.fine("Decontextualized object: " + jsonld); return jsonld; diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java index 09d02854bab..cf8b64520de 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonUtil.java @@ -11,6 +11,7 @@ import java.util.logging.Logger; import jakarta.json.Json; import jakarta.json.JsonArray; +import jakarta.json.JsonReader; import jakarta.json.JsonWriter; import jakarta.json.JsonWriterFactory; import jakarta.json.stream.JsonGenerator; @@ -60,13 +61,17 @@ public static String prettyPrint(jakarta.json.JsonObject jsonObject) { public static jakarta.json.JsonObject getJsonObject(String serializedJson) { try (StringReader rdr = new StringReader(serializedJson)) { - return Json.createReader(rdr).readObject(); + try (JsonReader jsonReader = Json.createReader(rdr)) { + return jsonReader.readObject(); + } } } public static jakarta.json.JsonArray getJsonArray(String serializedJson) { try (StringReader rdr = new StringReader(serializedJson)) { - return Json.createReader(rdr).readArray(); + try (JsonReader jsonReader = Json.createReader(rdr)) { + return jsonReader.readArray(); + } } } } diff --git a/src/main/webapp/file-download-popup-fragment.xhtml b/src/main/webapp/file-download-popup-fragment.xhtml index e1020c85e69..632c2a827ef 100644 --- a/src/main/webapp/file-download-popup-fragment.xhtml +++ b/src/main/webapp/file-download-popup-fragment.xhtml @@ -8,6 +8,8 @@ xmlns:jsf="http://xmlns.jcp.org/jsf" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> + +

@@ -51,6 +53,7 @@

+
+
\ No newline at end of file