Merge branch 'develop' into IQSS/3818_delete_thumbnail_tmp_files

.env

@@ -1,4 +1,4 @@
 APP_IMAGE=gdcc/dataverse:unstable
 POSTGRES_VERSION=13
 DATAVERSE_DB_USER=dataverse
-SOLR_VERSION=8.11.1
+SOLR_VERSION=9.3.0

.gitattributes

@@ -0,0 +1,4 @@
+# https://www.git-scm.com/docs/gitattributes
+
+# This set mandatory LF line endings for .sh files preventing from windows users to having to change the value of their git config --global core.autocrlf to 'false' or 'input'
+*.sh text eol=lf

.github/ISSUE_TEMPLATE/bug_report.md

@@ -3,7 +3,7 @@ name: Bug report
 about: Did you encounter something unexpected or incorrect in the Dataverse software?
   We'd like to hear about it!
 title: ''
-labels: ''
+labels: 'Type: Bug'
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea or new feature for the Dataverse software!
 title: 'Feature Request/Idea:'
-labels: ''
+labels: 'Type: Feature'
 assignees: ''
 
 ---

.github/workflows/container_app_pr.yml

@@ -25,7 +25,7 @@ jobs:
                   ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
             - uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: 'adopt'
             - uses: actions/cache@v3
               with:

.github/workflows/container_app_push.yml

@@ -36,10 +36,10 @@ jobs:
             - name: Checkout repository
               uses: actions/checkout@v3
 
-            - name: Set up JDK 11
+            - name: Set up JDK
               uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: temurin
                   cache: maven
 
@@ -99,17 +99,21 @@ jobs:
         name: "Package & Publish"
         runs-on: ubuntu-latest
         # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
-        # context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
-        if: needs.check-secrets.outputs.available == 'true'
+        # context of the main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
+        #
+        # Note: The team's decision was to not auto-deploy an image on any git push where no PR exists (yet).
+        #       Accordingly, only run for push events on branches develop and master.
+        if: needs.check-secrets.outputs.available == 'true' &&
+            ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name)))
         steps:
             - uses: actions/checkout@v3
             - uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: temurin
 
             # Depending on context, we push to different targets. Login accordingly.
-            - if: ${{ github.event_name != 'pull_request' }}
+            - if: github.event_name != 'pull_request'
               name: Log in to Docker Hub registry
               uses: docker/login-action@v2
               with:

.github/workflows/container_base_push.yml

@@ -34,7 +34,7 @@ jobs:
             packages: read
         strategy:
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
         # Only run in upstream repo - avoid unnecessary runs in forks
         if: ${{ github.repository_owner == 'IQSS' }}
 

.github/workflows/cypress_ui.yml.future

@@ -2,6 +2,7 @@
 #
 #   THIS IS AN OLD TRAVIS-CI.ORG JOB FILE
 #   To be used with Github Actions, it would be necessary to refactor it.
+#   In addition, it needs to be rewritten to use our modern containers.
 #   Keeping it as the future example it has been before.
 #   See also #5846
 #
@@ -30,8 +31,6 @@ jobs:
         directories:
           # we also need to cache folder with Cypress binary
           - ~/.cache
-          # we want to cache the Glassfish and Solr dependencies as well
-          - conf/docker-aio/dv/deps
       before_install:
         - cd tests
       install:

.github/workflows/deploy_beta_testing.yml

@@ -0,0 +1,85 @@
+name: 'Deploy to Beta Testing'
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+
+      - name: Enable API Session Auth feature flag
+        working-directory: src/main/resources/META-INF
+        run: echo -e "dataverse.feature.api-session-auth=true" >> microprofile-config.properties
+
+      - name: Set build number
+        run: scripts/installer/custom-build-number
+
+      - name: Build application war
+        run: mvn package
+
+      - name: Get war file name
+        working-directory: target
+        run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
+
+      - name: Upload war artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: built-app
+          path: ./target/${{ env.war_file }}
+
+  deploy-to-payara:
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download war artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: built-app
+          path: ./
+
+      - name: Get war file name
+        run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
+
+      - name: Copy war file to remote instance
+        uses: appleboy/scp-action@master
+        with:
+          host: ${{ secrets.BETA_PAYARA_INSTANCE_HOST }}
+          username: ${{ secrets.BETA_PAYARA_INSTANCE_USERNAME }}
+          key: ${{ secrets.BETA_PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
+          source: './${{ env.war_file }}'
+          target: '/home/${{ secrets.BETA_PAYARA_INSTANCE_USERNAME }}'
+          overwrite: true
+
+      - name: Execute payara war deployment remotely
+        uses: appleboy/[email protected]
+        env:
+          INPUT_WAR_FILE: ${{ env.war_file }}
+        with:
+          host: ${{ secrets.BETA_PAYARA_INSTANCE_HOST }}
+          username: ${{ secrets.BETA_PAYARA_INSTANCE_USERNAME }}
+          key: ${{ secrets.BETA_PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
+          envs: INPUT_WAR_FILE
+          script: |
+            APPLICATION_NAME=dataverse-backend
+            ASADMIN='/usr/local/payara6/bin/asadmin --user admin'
+            $ASADMIN undeploy $APPLICATION_NAME
+            $ASADMIN stop-domain
+            rm -rf /usr/local/payara6/glassfish/domains/domain1/generated
+            rm -rf /usr/local/payara6/glassfish/domains/domain1/osgi-cache
+            $ASADMIN start-domain
+            $ASADMIN deploy --name $APPLICATION_NAME $INPUT_WAR_FILE
+            $ASADMIN stop-domain
+            $ASADMIN start-domain

.github/workflows/maven_unit_test.yml

@@ -1,4 +1,4 @@
-name: Maven Unit Tests
+name: Maven Tests
 
 on:
     push:
@@ -22,21 +22,13 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
                 experimental: [false]
                 status:  ["Stable"]
-                #
-                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
-                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
-                # (This is a limitation of how Github is currently handling these things.)
-                #
-                #include:
-                #    - jdk: '17'
-                #      experimental: true
-                #      status: "Experimental"
         continue-on-error: ${{ matrix.experimental }}
         runs-on: ubuntu-latest
         steps:
+          # Basic setup chores
           - uses: actions/checkout@v3
           - name: Set up JDK ${{ matrix.jdk }}
             uses: actions/setup-java@v3
@@ -46,16 +38,110 @@ jobs:
                 cache: maven
 
           # The reason why we use "install" here is that we want the submodules to be available in the next step.
-          # Also, we can cache them this way for jobs triggered by this one.
-          - name: Build with Maven
+          # Also, we can cache them this way for jobs triggered by this one. We need to skip ITs here, as we run
+          # them in the next job - but install usually runs through verify phase.
+          - name: Build with Maven and run unit tests
             run: > 
                 mvn -B -f modules/dataverse-parent
                 -Dtarget.java.version=${{ matrix.jdk }}
                 -DcompilerArgument=-Xlint:unchecked -P all-unit-tests
+                -DskipIntegrationTests
                 -pl edu.harvard.iq:dataverse -am
                 install
 
-          - name: Maven Code Coverage
+          # We don't want to cache the WAR file, so delete it
+          - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
+
+          # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
+          # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
+          - uses: actions/upload-artifact@v3
+            with:
+                name: dataverse-java${{ matrix.jdk }}.war
+                path: target/dataverse*.war
+                retention-days: 7
+
+          # Store the build for the next step (integration test) to avoid recompilation and to transfer coverage reports
+          - run: |
+                tar -cvf java-builddir.tar target
+                tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-*
+          - uses: actions/upload-artifact@v3
+            with:
+                name: java-artifacts
+                path: |
+                    java-builddir.tar
+                    java-m2-selection.tar
+                retention-days: 3
+
+    integration-test:
+        runs-on: ubuntu-latest
+        needs: unittest
+        name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Integration Tests
+        strategy:
+            fail-fast: false
+            matrix:
+                jdk: [ '17' ]
+                experimental: [ false ]
+                status: [ "Stable" ]
+                #
+                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
+                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
+                # (This is a limitation of how Github is currently handling these things.)
+                #
+                #include:
+                #    - jdk: '17'
+                #      experimental: true
+                #      status: "Experimental"
+        continue-on-error: ${{ matrix.experimental }}
+        steps:
+            # Basic setup chores
+            - uses: actions/checkout@v3
+            - name: Set up JDK ${{ matrix.jdk }}
+              uses: actions/setup-java@v3
+              with:
+                  java-version: ${{ matrix.jdk }}
+                  distribution: temurin
+                  cache: maven
+
+            # Get the build output from the unit test job
+            - uses: actions/download-artifact@v3
+              with:
+                  name: java-artifacts
+            - run: |
+                tar -xvf java-builddir.tar
+                tar -xvf java-m2-selection.tar -C /
+
+            # Run integration tests (but not unit tests again)
+            - run: mvn -DskipUnitTests -Dtarget.java.version=${{ matrix.jdk }} verify
+
+            # Wrap up and send to coverage job
+            - run: tar -cvf java-reportdir.tar target/site
+            - uses: actions/upload-artifact@v3
+              with:
+                  name: java-reportdir
+                  path: java-reportdir.tar
+                  retention-days: 3
+
+    coverage-report:
+        runs-on: ubuntu-latest
+        needs: integration-test
+        name: Coverage Report Submission
+        steps:
+            # Basic setup chores
+          - uses: actions/checkout@v3
+          - uses: actions/setup-java@v3
+            with:
+                java-version: '17'
+                distribution: temurin
+                cache: maven
+
+          # Get the build output from the integration test job
+          - uses: actions/download-artifact@v3
+            with:
+                name: java-reportdir
+          - run: tar -xvf java-reportdir.tar
+
+          # Deposit Code Coverage
+          - name: Deposit Code Coverage
             env:
                 CI_NAME: github
                 COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
@@ -66,14 +152,14 @@ jobs:
                 -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
                 jacoco:report coveralls:report
 
-          # We don't want to cache the WAR file, so delete it
-          - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
+          # NOTE: this may be extended with adding a report to the build output, leave a comment, send to Sonarcloud, ...
+
     push-app-img:
         name: Publish App Image
         permissions:
             contents: read
             packages: write
             pull-requests: write
-        needs: unittest
+        needs: integration-test
         uses: ./.github/workflows/container_app_push.yml
         secrets: inherit

.github/workflows/shellcheck.yml

@@ -33,7 +33,6 @@ jobs:
                   # Exclude old scripts
                   exclude: |
                       */.git/*
-                      conf/docker-aio/*
                       doc/*
                       downloads/*
                       scripts/database/*
@@ -43,5 +42,4 @@ jobs:
                       scripts/issues/*
                       scripts/r/*
                       scripts/tests/*
-                      scripts/vagrant/*
                       tests/*

.github/workflows/shellspec.yml

@@ -60,7 +60,7 @@ jobs:
                   shellspec
     shellspec-macos:
         name: "MacOS"
-        runs-on: macos-10.15
+        runs-on: macos-latest
         steps:
             - name: Install shellspec
               run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes