Merge pull request #7761 from IQSS/develop

v5.4

.gitignore

@@ -46,6 +46,10 @@ conf/docker-aio/dv/deps/
 conf/docker-aio/dv/install/dvinstall.zip
 # or copy of test data
 conf/docker-aio/testdata/
+
+# docker-aio creates maven/ which reports 86 new files. ignore this wd.
+maven/
+
 scripts/installer/default.config
 *.pem
 

.travis.yml

@@ -1,5 +1,5 @@
 language: java
 jdk:
-  - openjdk8
+  - openjdk11
 script: mvn -DcompilerArgument=-Xlint:unchecked test -P all-unit-tests
 after_success: mvn jacoco:report coveralls:report 

conf/docker-aio/0prep_deps.sh

@@ -10,12 +10,11 @@ if [ ! -e dv/deps/payara-5.2020.6.zip ]; then
 	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2020.6/payara-5.2020.6.zip  -O dv/deps/payara-5.2020.6.zip
 fi
 
-if [ ! -e dv/deps/solr-7.7.2dv.tgz ]; then
+if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then
 	echo "solr dependency prep"
 	# schema changes *should* be the only ones...
-	cd dv/deps/
-	#wget https://archive.apache.org/dist/lucene/solr/7.3.0/solr-7.3.0.tgz -O solr-7.3.0dv.tgz
-	wget https://archive.apache.org/dist/lucene/solr/7.7.2/solr-7.7.2.tgz -O solr-7.7.2dv.tgz
+	cd dv/deps/	
+	wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz
 	cd ../../
 fi
 

conf/docker-aio/1prep.sh

@@ -1,22 +1,32 @@
-#!/bin/sh
+#!/bin/bash
 
 # move things necessary for integration tests into build context.
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
 
 mkdir -p testdata/doc/sphinx-guides/source/_static/util/
-cp ../solr/7.7.2/schema*.xml testdata/
-cp ../solr/7.7.2/solrconfig.xml testdata/
-cp ../solr/7.7.2/updateSchemaMDB.sh testdata/
+cp ../solr/8.8.1/schema*.xml testdata/
+cp ../solr/8.8.1/solrconfig.xml testdata/
+cp ../solr/8.8.1/updateSchemaMDB.sh testdata/
 cp ../jhove/jhove.conf testdata/
 cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../
 cp -r scripts conf/docker-aio/testdata/
 cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
 
+wget -q https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
+tar xfz apache-maven-3.6.3-bin.tar.gz
+mkdir maven
+mv apache-maven-3.6.3/* maven/
+echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" > maven/maven.sh
+echo "export M2_HOME=../maven" >> maven/maven.sh
+echo "export MAVEN_HOME=../maven" >> maven/maven.sh
+echo "export PATH=../maven/bin:${PATH}" >> maven/maven.sh
+chmod 0755 maven/maven.sh
+
 # not using dvinstall.zip for setupIT.bash; but still used in install.bash for normal ops
-mvn clean
+source maven/maven.sh && mvn clean
 ./scripts/installer/custom-build-number
-mvn package
+source maven/maven.sh && mvn package
 cd scripts/installer
 make clean
 make

conf/docker-aio/c8.dockerfile

@@ -2,7 +2,7 @@ FROM centos:8
 # OS dependencies
 # PG 10 is the default in centos8; keep the repo comment for when we bump to 11+
 #RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm
-RUN yum install -y java-1.8.0-openjdk-devel postgresql-server sudo epel-release unzip perl curl httpd
+RUN yum install -y java-11-openjdk-devel postgresql-server sudo epel-release unzip curl httpd
 RUN yum install -y jq lsof awscli
 
 # copy and unpack dependencies (solr, payara)
@@ -17,20 +17,20 @@ COPY testdata/sushi_sample_logs.json /tmp/
 COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
-RUN cd /opt ; tar zxf /tmp/dv/deps/solr-7.7.2dv.tgz 
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz 
 RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2020.6.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
-COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml
+# this dies under Java 11, do we keep it?
+#COPY domain-restmonitor.xml /opt/payara5/glassfish/domains/domain1/config/domain.xml
 
-#RUN sudo -u postgres /usr/bin/initdb -D /var/lib/pgsql/data
-RUN sudo -u postgres /usr/pgsql-9.6/bin/initdb -D /var/lib/pgsql/data
+RUN sudo -u postgres /usr/bin/initdb /var/lib/pgsql/data
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/data/
-RUN cp -r /opt/solr-7.7.2/server/solr/configsets/_default /opt/solr-7.7.2/server/solr/collection1
-RUN cp /tmp/dv/schema*.xml /opt/solr-7.7.2/server/solr/collection1/conf/
-RUN cp /tmp/dv/solrconfig.xml /opt/solr-7.7.2/server/solr/collection1/conf/solrconfig.xml
+RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1
+RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/
+RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml
 
 # skipping payara user and solr user (run both as root)
 

conf/docker-aio/entrypoint.bash

@@ -1,8 +1,7 @@
 #!/usr/bin/env bash
 export LANG=en_US.UTF-8
-#sudo -u postgres /usr/bin/postgres -D /var/lib/pgsql/data &
-sudo -u postgres /usr/pgsql-9.6/bin/postgres -D /var/lib/pgsql/data &
-cd /opt/solr-7.7.2/
+sudo -u postgres /usr/bin/pg_ctl start -D /var/lib/pgsql/data &
+cd /opt/solr-8.8.1/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force
 bin/solr create_core -c collection1 -d server/solr/collection1/conf -force

conf/docker-aio/readme.md

@@ -4,7 +4,7 @@ First pass docker all-in-one image, intended for running integration tests again
 Also usable for normal development and system evaluation; not intended for production.
 
 ### Requirements:
- - java8 compiler, maven, make, wget, docker
+ - java11 compiler, maven, make, wget, docker
 
 ### Quickstart:
  - in the root of the repository, run `./conf/docker-aio/prep_it.bash`

conf/docker-aio/run-test-suite.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # This is the canonical list of which "IT" tests are expected to pass.
 
 dvurl=$1
@@ -8,4 +8,4 @@ fi
 
 # Please note the "dataverse.test.baseurl" is set to run for "all-in-one" Docker environment.
 # TODO: Rather than hard-coding the list of "IT" classes here, add a profile to pom.xml.
-mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT -Ddataverse.test.baseurl=$dvurl
+source maven/maven.sh && mvn test -Dtest=DataversesIT,DatasetsIT,SwordIT,AdminIT,BuiltinUsersIT,UsersIT,UtilIT,ConfirmEmailIT,FileMetadataIT,FilesIT,SearchIT,InReviewWorkflowIT,HarvestingServerIT,MoveIT,MakeDataCountApiIT,FileTypeDetectionIT,EditDDIIT,ExternalToolsIT,AccessIT,DuplicateFilesIT,DownloadFilesIT,LinkIT,DeleteUsersIT,DeactivateUsersIT -Ddataverse.test.baseurl=$dvurl

conf/docker-aio/testscripts/db.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-psql -U postgres -c "CREATE ROLE dvnapp UNENCRYPTED PASSWORD 'secret' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN" template1
+psql -U postgres -c "CREATE ROLE dvnapp PASSWORD 'secret' SUPERUSER CREATEDB CREATEROLE INHERIT LOGIN" template1
 psql -U dvnapp -c 'CREATE DATABASE "dvndb" WITH OWNER = "dvnapp"' template1

conf/solr/7.7.2/schema.xml → conf/solr/8.8.1/schema.xml

@@ -38,7 +38,7 @@
     catchall "text" field, and use that for searching.
 -->
 
-<schema name="default-config" version="1.6">
+<schema name="default-config" version="1.7">
     <!-- attribute "name" is the name of this schema and is only used for display purposes.
        version="x.y" is Solr's version number for the schema syntax and 
        semantics.  It should not normally be changed by applications.
@@ -566,6 +566,7 @@
             <filter class="solr.KeywordRepeatFilterFactory" />
             <filter class="solr.PorterStemFilterFactory"/>
             <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+            <filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true" />
         </analyzer>
         <analyzer type="query">
             <tokenizer class="solr.StandardTokenizerFactory"/>
@@ -616,6 +617,7 @@
         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
         <filter class="solr.PorterStemFilterFactory"/>
         <filter class="solr.FlattenGraphFilterFactory" />
+        <filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="true" />
       </analyzer>
       <analyzer type="query">
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>

doc/release-notes/5.0-release-notes.md

@@ -245,36 +245,58 @@ The instructions below describe the upgrade procedure based on moving an existin
    `sudo /usr/local/glassfish4/bin/asadmin stop-domain`
    `sudo cp -ar /usr/local/glassfish4/glassfish/domains/domain1 /usr/local/payara5/glassfish/domains/`
 
-7. Remove the Glassfish cache directories
+7. Remove the cache directories
 
-   `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/`  `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/`
+   `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/generated/`  
+   `sudo rm -rf /usr/local/payara5/glassfish/domains/domain1/osgi-cache/`
 
-8. In domain.xml:
+8. Make the following changes in **domain.xml**:
 
- Replace the -XX:PermSize and -XX:MaxPermSize JVM options with -XX:MetaspaceSize and -XX:MaxMetaspaceSize.
+   Replace the `-XX:PermSize` and `-XX:MaxPermSize` JVM options with `-XX:MetaspaceSize` and `-XX:MaxMetaspaceSize`
 
+   ```
    <jvm-options>-XX:MetaspaceSize=256m</jvm-options>
    <jvm-options>-XX:MaxMetaspaceSize=512m</jvm-options>
+   ```
 
-Add the below JVM options beneath the -Ddataverse settings:  
+   Add the below JVM options beneath the -Ddataverse settings:  
 
+   ```
    <jvm-options>-Dfish.payara.classloading.delegate=false</jvm-options>
    <jvm-options>-XX:+UseG1GC</jvm-options>
    <jvm-options>-XX:+UseStringDeduplication</jvm-options>
    <jvm-options>-XX:+DisableExplicitGC</jvm-options>
+   ```
 
-9. Change any full pathnames /usr/local/glassfish4/... to /usr/local/payara5/... or whatever it is in your case. (Specifically check the -Ddataverse.files.directory and -Ddataverse.files.file.directory JVM options)
+   Replace the following element:
 
-10.  In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above.
+   ```
+   <jdbc-connection-pool datasource-classname="org.apache.derby.jdbc.EmbeddedXADataSource" name="__TimerPool" res-type="javax.sql.XADataSource">
+      <property name="databaseName" value="${com.sun.aas.instanceRoot}/lib/databases/ejbtimer"></property>
+      <property name="connectionAttributes" value=";create=true"></property>
+   </jdbc-connection-pool>
+   ```
 
-(Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`:
+   with
+
+  ```
+  <jdbc-connection-pool datasource-classname="org.h2.jdbcx.JdbcDataSource" name="__TimerPool" res-type="javax.sql.XADataSource">
+    <property name="URL" value="jdbc:h2:${com.sun.aas.instanceRoot}/lib/databases/ejbtimer;AUTO_SERVER=TRUE"></property>
+  </jdbc-connection-pool>
+  ```
+
+9. Change any full pathnames matching `/usr/local/glassfish4/...` to `/usr/local/payara5/...` or whatever it is in your case. Specifically check the `-Ddataverse.files.directory` and `-Ddataverse.files.file.directory` JVM options.
+
+10.   In domain1/config/jhove.conf, change the hard-coded /usr/local/glassfish4 path, as above.
+
+   (Optional): If you renamed your service account from glassfish to payara or appserver, update the ownership permissions. The Installation Guide recommends a service account of `dataverse`:
 
    `sudo chown -R dataverse /usr/local/payara5/glassfish/domains/domain1`
    `sudo chown -R dataverse /usr/local/payara5/glassfish/lib`
 
-11. You will also need to check that the service account has write permission on the files directory, if they are located outside the old Glassfish domain. And/or make sure the service account has the correct AWS credentials, if you are using S3 for storage.
+11.  You will also need to check that the service account has write permission on the files directory, if they are located outside the old Glassfish domain. And/or make sure the service account has the correct AWS credentials, if you are using S3 for storage.
 
-12. Finally, start Payara:
+12.  Finally, start Payara:
 
    `sudo -u dataverse /usr/local/payara5/bin/asadmin start-domain`
 
@@ -300,20 +322,20 @@ Add the below JVM options beneath the -Ddataverse settings:
 
 3. (Required for installations using DataCite) Add the JVM option doi.dataciterestapiurlstring
 
-For production environments:
+   For production environments:
 
    `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.datacite.org"`
 
-For test environments: 
+   For test environments: 
 
    `/usr/local/payara5/bin/asadmin create-jvm-options "\-Ddoi.dataciterestapiurlstring=https\://api.test.datacite.org"`
 
-The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example:
+   The JVM option `doi.mdcbaseurlstring` should be deleted if it was previously set, for example:
 
    `/usr/local/payara5/bin/asadmin delete-jvm-options "\-Ddoi.mdcbaseurlstring=https\://api.test.datacite.org"`
 
 4. (Recommended for installations using DataCite) Pre-register DOIs
 
-Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. 
+   Execute the script described in the section "Dataverse Installations Using DataCite: Upgrade Action Recommended" earlier in the Release Note. 
 
-Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. 
+   Please consult the earlier sections of the Release Note for any additional configuration options that may apply to your installation. 

doc/release-notes/5.3-release-notes.md

@@ -104,12 +104,15 @@ If this is a new installation, please see our [Installation Guide](https://guide
 
 ## Upgrade Instructions
 
-0\. These instructions assume that you've already successfully upgraded from Dataverse 4.x to  Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0).
+0\. These instructions assume that you've already successfully upgraded from Dataverse 4.x to Dataverse 5 following the instructions in the [Dataverse 5 Release Notes](https://github.com/IQSS/dataverse/releases/tag/v5.0).
 
 1\. Upgrade to Payara 5.2020.6 or higher.
 
-Instructions how to update can be found in the
-[Payara documentation](https://docs.payara.fish/community/docs/5.2020.6/documentation/user-guides/upgrade-payara.html)
+Instructions on how to update can be found in the
+[Payara documentation](https://docs.payara.fish/community/docs/5.2020.6/documentation/user-guides/upgrade-payara.html).
+
+It would likely be safer to upgrade Payara first, while still running Dataverse 5.2, and then proceed with the steps below. Upgrading from an earlier version of Payara should be a straightforward process: Undeploy Dataverse; stop Payara; move the current Payara directory out of the way; unzip the new Payara version in its place; replace **the brand new `payara/glassfish/domains/domain1` with your old, preserved `domain1`**; start Payara, deploy Dataverse 5.2. We still recommend that you read the detailed upgrade instructions above; and, if you run into any issues with this upgrade, it will help to be able to separate them from any problems with the upgrade of Dataverse proper. 
+If you are still using pre-5.0 version of Dataverse, and Glassfish version 4, please follow the upgrade instructions in the Dataverse 5.0 release notes; but use the latest version of Payara 5 (5.2020.7, as of this writing).
 
 2\. Undeploy the previous version.
 
@@ -132,27 +135,31 @@ Please configure your connection details, replacing all the `${DB_...}`.
 - `<payara install path>/bin/asadmin create-password-alias --passwordfile /tmp/password.txt dataverse.db.password`
 - `rm /tmp/password.txt`
 
-4\. Reset the EJB timer database back to default:
+4\. In domain.xml, verify that the __TimerPool jdbc-connection-pool is using the H2 database, as follows (if you have the old Derby version from Glassfish 4, replace it):
+
+  `<jdbc-connection-pool datasource-classname="org.h2.jdbcx.JdbcDataSource" name="__TimerPool" res-type="javax.sql.XADataSource"> <property name="URL" value="jdbc:h2:${com.sun.aas.instanceRoot}/lib/databases/ejbtimer;AUTO_SERVER=TRUE"></property> </jdbc-connection-pool>`
+
+5\. Reset the EJB timer database back to default:
 
 - `<payara install path>/bin/asadmin set configs.config.server-config.ejb-container.ejb-timer-service.timer-datasource=jdbc/__TimerPool`
 
-5\. Delete the old password alias and DB pool:
+6\. Delete the old password alias and DB pool:
 
 - `<payara install path>/bin/asadmin delete-jdbc-connection-pool --cascade=true dvnDbPool`
 - `<payara install path>/bin/asadmin delete-password-alias db_password_alias`
 
-6\. Stop payara and remove the generated directory, start.
+7\. Stop payara, remove the generated and ejbtimer database directories, then restart.
 
 - `service payara stop`
-- remove the generated directory:
-`rm -rf <payara install path>/glassfish/domains/domain1/generated`
+- `rm -rf <payara install path>/glassfish/domains/domain1/generated`
+- `rm -rf <payara install path>/glassfish/domains/domain1/lib/databases/ejbtimer`
 - `service payara start`
 
-7\. Deploy this version.
+8\. Deploy this version.
 
 - `<payara install path>/bin/asadmin deploy dataverse-5.3.war`
 
-8\. Restart payara
+9\. Restart payara
 
 - `service payara stop`
 - `service payara start`