Merge branch 'develop' of github.com:IQSS/dataverse into 9714-files-a…

…pi-extension-filters

.github/workflows/container_app_pr.yml

@@ -25,7 +25,7 @@ jobs:
                   ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
             - uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: 'adopt'
             - uses: actions/cache@v3
               with:

.github/workflows/container_app_push.yml

@@ -36,10 +36,10 @@ jobs:
             - name: Checkout repository
               uses: actions/checkout@v3
 
-            - name: Set up JDK 11
+            - name: Set up JDK
               uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: temurin
                   cache: maven
 
@@ -109,7 +109,7 @@ jobs:
             - uses: actions/checkout@v3
             - uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: temurin
 
             # Depending on context, we push to different targets. Login accordingly.

.github/workflows/container_base_push.yml

@@ -34,7 +34,7 @@ jobs:
             packages: read
         strategy:
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
         # Only run in upstream repo - avoid unnecessary runs in forks
         if: ${{ github.repository_owner == 'IQSS' }}
 

.github/workflows/deploy_beta_testing.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/setup-java@v3
         with:
           distribution: 'zulu'
-          java-version: '11'
+          java-version: '17'
 
       - name: Enable API Session Auth feature flag
         working-directory: src/main/resources/META-INF

.github/workflows/maven_unit_test.yml

@@ -22,18 +22,9 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
                 experimental: [false]
                 status:  ["Stable"]
-                #
-                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
-                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
-                # (This is a limitation of how Github is currently handling these things.)
-                #
-                #include:
-                #    - jdk: '17'
-                #      experimental: true
-                #      status: "Experimental"
         continue-on-error: ${{ matrix.experimental }}
         runs-on: ubuntu-latest
         steps:
@@ -68,6 +59,14 @@ jobs:
 
           # We don't want to cache the WAR file, so delete it
           - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
+
+          # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
+          # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
+          - uses: actions/upload-artifact@v3
+            with:
+                name: dataverse-java${{ matrix.jdk }}.war
+                path: target/dataverse*.war
+                retention-days: 7
     push-app-img:
         name: Publish App Image
         permissions:

.github/workflows/spi_release.yml

@@ -40,7 +40,7 @@ jobs:
             - uses: actions/checkout@v3
             - uses: actions/setup-java@v3
               with:
-                  java-version: '11'
+                  java-version: '17'
                   distribution: 'adopt'
                   server-id: ossrh
                   server-username: MAVEN_USERNAME
@@ -66,7 +66,7 @@ jobs:
             -   uses: actions/checkout@v3
             -   uses: actions/setup-java@v3
                 with:
-                    java-version: '11'
+                    java-version: '17'
                     distribution: 'adopt'
             -   uses: actions/cache@v2
                 with:
@@ -78,7 +78,7 @@ jobs:
             -   name: Set up Maven Central Repository
                 uses: actions/setup-java@v3
                 with:
-                    java-version: '11'
+                    java-version: '17'
                     distribution: 'adopt'
                     server-id: ossrh
                     server-username: MAVEN_USERNAME
@@ -91,4 +91,4 @@ jobs:
                 env:
                     MAVEN_USERNAME: ${{ secrets.DATAVERSEBOT_SONATYPE_USERNAME }}
                     MAVEN_PASSWORD: ${{ secrets.DATAVERSEBOT_SONATYPE_TOKEN }}
-                    MAVEN_GPG_PASSPHRASE: ${{ secrets.DATAVERSEBOT_GPG_PASSWORD }}
+                    MAVEN_GPG_PASSPHRASE: ${{ secrets.DATAVERSEBOT_GPG_PASSWORD }}

doc/release-notes/8094-java-17.md

@@ -0,0 +1 @@
+Java 17 or higher is now required.

doc/release-notes/9340-payara5to6.md

@@ -0,0 +1,122 @@
+## Upgrade from Payara 5 to Payara 6
+
+1. Download Payara 6.2023.8 as of this writing:
+
+   `curl -L -O https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip`
+
+1. Unzip it to /usr/local (or your preferred location):
+
+   `sudo unzip payara-6.2023.8.zip -d /usr/local/`
+
+1. Change ownership of the unzipped Payara to your "service" user ("dataverse" by default):
+
+   `sudo chown -R dataverse /usr/local/payara6`
+
+1. Undeploy Dataverse (if deployed, using the unprivileged service account. Version 5.14 is assumed in the example below):
+
+   `sudo -u dataverse /usr/local/payara5/bin/asadmin list-applications`
+
+   `sudo -u dataverse /usr/local/payara5/bin/asadmin undeploy dataverse-5.14`
+
+1. Stop Payara 5:
+
+   `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain`
+
+1. Copy Dataverse-related lines from Payara 5 to Payara 6 domain.xml:
+
+   `sudo -u dataverse cp /usr/local/payara6/glassfish/domains/domain1/config/domain.xml /usr/local/payara6/glassfish/domains/domain1/config/domain.xml.orig`
+
+   `sudo egrep 'dataverse|doi' /usr/local/payara5/glassfish/domains/domain1/config/domain.xml > lines.txt`
+
+   `sudo vi /usr/local/payara6/glassfish/domains/domain1/config/domain.xml`
+
+   The lines will appear in two sections, examples shown below (but your content will vary).
+
+   Section 1: system properties (under `<server name="server" config-ref="server-config">`)
+
+   ```
+   <system-property name="dataverse.db.user" value="dvnuser"></system-property>
+   <system-property name="dataverse.db.host" value="localhost"></system-property>
+   <system-property name="dataverse.db.port" value="5432"></system-property>
+   <system-property name="dataverse.db.name" value="dvndb"></system-property>
+   <system-property name="dataverse.db.password" value="dvnsecret"></system-property>
+   ```
+
+   Note: if you used the Dataverse installer, you won't have a `dataverse.db.password` property. See "Create password aliases" below.
+
+   Section 2: JVM options (under `<java-config classpath-suffix="" debug-options="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=9009" system-classpath="">`, the one under `<config name="server-config">`, not under `<config name="default-config">`)
+
+   ```
+   <jvm-options>-Ddataverse.files.directory=/usr/local/dvn/data</jvm-options>
+   <jvm-options>-Ddataverse.files.file.type=file</jvm-options>
+   <jvm-options>-Ddataverse.files.file.label=file</jvm-options>
+   <jvm-options>-Ddataverse.files.file.directory=/usr/local/dvn/data</jvm-options>
+   <jvm-options>-Ddataverse.rserve.host=localhost</jvm-options>
+   <jvm-options>-Ddataverse.rserve.port=6311</jvm-options>
+   <jvm-options>-Ddataverse.rserve.user=rserve</jvm-options>
+   <jvm-options>-Ddataverse.rserve.password=rserve</jvm-options>
+   <jvm-options>-Ddataverse.auth.password-reset-timeout-in-minutes=60</jvm-options>
+   <jvm-options>-Ddataverse.timerServer=true</jvm-options>
+   <jvm-options>-Ddataverse.fqdn=dev1.dataverse.org</jvm-options>
+   <jvm-options>-Ddataverse.siteUrl=https://dev1.dataverse.org</jvm-options>
+   <jvm-options>-Ddataverse.files.storage-driver-id=file</jvm-options>
+   <jvm-options>-Ddoi.username=testaccount</jvm-options>
+   <jvm-options>-Ddoi.password=notmypassword</jvm-options>
+   <jvm-options>-Ddoi.baseurlstring=https://mds.test.datacite.org/</jvm-options>
+   <jvm-options>-Ddoi.dataciterestapiurlstring=https://api.test.datacite.org</jvm-options>
+   ```
+
+1. Check the `Xmx` setting in `/usr/local/payara6/glassfish/domains/domain1/config/domain.xml`. (The one under `<config name="server-config">`, where you put the JVM options, not the one under `<config name="default-config">`.) Note that there are two such settings, and you want to adjust the one in the stanza with Dataverse options. This sets the JVM heap size; a good rule of thumb is half of your system's total RAM. You may specify the value in MB (`8192m`) or GB (`8g`).
+
+1. Copy jhove.conf and jhoveConfig.xsd from Payara 5, edit and change payara5 to payara6
+
+   `sudo cp /usr/local/payara5/glassfish/domains/domain1/config/jhove* /usr/local/payara6/glassfish/domains/domain1/config/`
+
+   `sudo chown dataverse /usr/local/payara6/glassfish/domains/domain1/config/jhove*`
+
+   `sudo -u dataverse vi /usr/local/payara6/glassfish/domains/domain1/config/jhove.conf`
+
+1. Update systemd unit file (or other init system) from `/usr/local/payara5` to `/usr/local/payara6`, if applicable.
+
+1. Start Payara:
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain`
+
+1. Create a Java mail resource, replacing "localhost" for mailhost with your mail relay server, and replacing "localhost" for fromaddress with the FQDN of your Dataverse server:
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin create-javamail-resource --mailhost "localhost" --mailuser "dataversenotify" --fromaddress "do-not-reply@localhost" mail/notifyMailSession`
+
+1. Create password aliases for your database, rserve and datacite jvm-options, if you're using them:
+
+   ```
+   $ echo "AS_ADMIN_ALIASPASSWORD=yourDBpassword" > /tmp/dataverse.db.password.txt
+   $ sudo -u dataverse /usr/local/payara6/bin/asadmin create-password-alias --passwordfile /tmp/dataverse.db.password.txt 
+   Enter the value for the aliasname operand> dataverse.db.password
+   Command create-password-alias executed successfully.
+   ```
+
+   You'll want to perform similar commands for `rserve_password_alias` and `doi_password_alias` if you're using Rserve and/or Datacite.
+
+1. Enable workaround for FISH-7722:
+
+   The following workaround is for https://github.com/payara/Payara/issues/6337
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED`
+
+1. Create the network listener on port 8009
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin create-network-listener --protocol http-listener-1 --listenerport 8009 --jkenabled true jk-connector`
+
+1. Deploy the Dataverse 6.0 warfile:
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin deploy /path/to/dataverse-6.0.war`
+
+1. Check that you get a version number from Dataverse:
+
+   `curl http://localhost:8080/api/info/version`
+
+1. Perform one final Payara restart to ensure that timers are initialized properly:
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin stop-domain`
+
+   `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain`

doc/release-notes/9782-juni5-transition.md

@@ -0,0 +1,7 @@
+# Migrating all test to JUnit 5
+With this release, we transition all of our test cases (see `src/test/`) to use JUnit 5 only.
+Moving forward from JUnit 4 will allow writing tests in more concise and easier ways.
+The tests themselves have not been altered, but updated to match JUnit 5 ways.
+They have not been extended or dropped coverage; this is mostly a preparation of things to come in the future. 
+If you are writing tests in JUnit 4 in your feature branches, you need to migrate.
+The development guides section of testing has been updated as well.

doc/sphinx-guides/source/container/base-image.rst

@@ -41,7 +41,7 @@ Image Contents
 
 The base image provides:
 
-- `Eclipse Temurin JRE using Java 11 <https://adoptium.net/temurin/releases?version=11>`_
+- `Eclipse Temurin JRE using Java 17 <https://adoptium.net/temurin/releases?version=17>`_
 - `Payara Community Application Server <https://docs.payara.fish/community>`_
 - CLI tools necessary to run Dataverse (i. e. ``curl`` or ``jq`` - see also :doc:`../installation/prerequisites` in Installation Guide)
 - Linux tools for analysis, monitoring and so on

doc/sphinx-guides/source/developers/classic-dev-env.rst

@@ -91,15 +91,15 @@ On Linux, install ``jq`` from your package manager or download a binary from htt
 Install Payara
 ~~~~~~~~~~~~~~
 
-Payara 6.2023.7 or higher is required.
+Payara 6.2023.8 or higher is required.
 
 To install Payara, run the following commands:
 
 ``cd /usr/local``
 
-``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.7/payara-6.2023.7.zip``
+``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip``
 
-``sudo unzip payara-6.2023.7.zip``
+``sudo unzip payara-6.2023.8.zip``
 
 ``sudo chown -R $USER /usr/local/payara6``
 

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -12,7 +12,7 @@ These instructions are oriented around Docker but the "classic" instructions we
 Quickstart
 ----------
 
-First, install Java 11, Maven, and Docker.
+First, install Java 17, Maven, and Docker.
 
 After cloning the `dataverse repo <https://github.com/IQSS/dataverse>`_, run this:
 
@@ -30,7 +30,7 @@ Detailed Steps
 Install Java
 ~~~~~~~~~~~~
 
-The Dataverse Software requires Java 11.
+The Dataverse Software requires Java 17.
 
 On Mac and Windows, we suggest downloading OpenJDK from https://adoptium.net (formerly `AdoptOpenJDK <https://adoptopenjdk.net>`_) or `SDKMAN <https://sdkman.io>`_.
 

doc/sphinx-guides/source/developers/testing.rst

@@ -47,12 +47,14 @@ Writing Unit Tests with JUnit
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 We are aware that there are newer testing tools such as TestNG, but we use `JUnit <http://junit.org>`_ because it's tried and true.
-We support both (legacy) JUnit 4.x tests (forming the majority of our tests) and
-newer JUnit 5 based testing.
+We support JUnit 5 based testing and require new tests written with it.
+(Since Dataverse 6.0, we migrated all of our tests formerly based on JUnit 4.)
 
-NOTE: When adding new tests, you should give JUnit 5 a go instead of adding more dependencies to JUnit 4.x.
-
-If writing tests is new to you, poke around existing unit tests which all end in ``Test.java`` and live under ``src/test``. Each test is annotated with ``@Test`` and should have at least one assertion which specifies the expected result. In Netbeans, you can run all the tests in it by clicking "Run" -> "Test File". From the test file, you should be able to navigate to the code that's being tested by right-clicking on the file and clicking "Navigate" -> "Go to Test/Tested class". Likewise, from the code, you should be able to use the same "Navigate" menu to go to the tests.
+If writing tests is new to you, poke around existing unit tests which all end in ``Test.java`` and live under ``src/test``.
+Each test is annotated with ``@Test`` and should have at least one assertion which specifies the expected result.
+In Netbeans, you can run all the tests in it by clicking "Run" -> "Test File".
+From the test file, you should be able to navigate to the code that's being tested by right-clicking on the file and clicking "Navigate" -> "Go to Test/Tested class".
+Likewise, from the code, you should be able to use the same "Navigate" menu to go to the tests.
 
 NOTE: Please remember when writing tests checking possibly localized outputs to check against ``en_US.UTF-8`` and ``UTC``
 l10n strings!
@@ -62,22 +64,24 @@ Refactoring Code to Make It Unit-Testable
 
 Existing code is not necessarily written in a way that lends itself to easy testing. Generally speaking, it is difficult to write unit tests for both JSF "backing" beans (which end in ``Page.java``) and "service" beans (which end in ``Service.java``) because they require the database to be running in order to test them. If service beans can be exercised via API they can be tested with integration tests (described below) but a good technique for making the logic testable it to move code to "util beans" (which end in ``Util.java``) that operate on Plain Old Java Objects (POJOs). ``PrivateUrlUtil.java`` is a good example of moving logic from ``PrivateUrlServiceBean.java`` to a "util" bean to make the code testable.
 
-Parameterized Tests and JUnit Theories
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Parameterized Tests
+^^^^^^^^^^^^^^^^^^^
+
 Often times you will want to test a method multiple times with similar values.
 In order to avoid test bloat (writing a test for every data combination),
 JUnit offers Data-driven unit tests. This allows a test to be run for each set
 of defined data values.
 
-JUnit 4 uses ``Parameterized.class`` and ``Theories.class``. For reference, take a look at issue https://github.com/IQSS/dataverse/issues/5619.
-
-JUnit 5 doesn't offer theories (see `jqwik <https://jqwik.net>`_ for this), but
-greatly extended parameterized testing. Some guidance how to write those:
+JUnit 5 offers great parameterized testing. Some guidance how to write those:
 
 - https://junit.org/junit5/docs/current/user-guide/#writing-tests-parameterized-tests
 - https://www.baeldung.com/parameterized-tests-junit-5
 - https://blog.codefx.org/libraries/junit-5-parameterized-tests/
-- See also some examples in our codebase.
+- See also many examples in our codebase.
+
+Note that JUnit 5 also offers support for custom test parameter resolvers. This enables keeping tests cleaner,
+as preparation might happen within some extension and the test code is more focused on the actual testing.
+See https://junit.org/junit5/docs/current/user-guide/#extensions-parameter-resolution for more information.
 
 JUnit 5 Test Helper Extensions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -116,11 +120,14 @@ In addition, there is a writeup on "The Testable Command" at https://github.com/
 Running Non-Essential (Excluded) Unit Tests
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Category(NonEssentialTests.class)`` and are excluded from the "dev" Maven profile, which is the default profile. All unit tests (that have not been annotated with ``@Ignore``), including these non-essential tests, are run from continuous integration systems such as Jenkins and GitHub Actions with the following ``mvn`` command that invokes a non-default profile:
+You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Tag(Tags.NOT_ESSENTIAL_UNITTESTS)`` and are excluded from the "dev" Maven profile, which is the default profile.
+All unit tests (that have not been annotated with ``@Disable``), including these non-essential tests, are run from continuous integration systems such as Jenkins and GitHub Actions with the following ``mvn`` command that invokes a non-default profile:
 
 ``mvn test -P all-unit-tests``
 
-Generally speaking, unit tests have been flagged as non-essential because they are slow or because they require an Internet connection. You should not feel obligated to run these tests continuously but you can use the ``mvn`` command above to run them. To iterate on the unit test in Netbeans and execute it with "Run -> Test File", you must temporarily comment out the annotation flagging the test as non-essential.
+Generally speaking, unit tests have been flagged as non-essential because they are slow or because they require an Internet connection.
+You should not feel obligated to run these tests continuously but you can use the ``mvn`` command above to run them.
+To iterate on the unit test in Netbeans and execute it with "Run -> Test File", you must temporarily comment out the annotation flagging the test as non-essential.
 
 Integration Tests
 -----------------