Merge branch 'develop' into 9361-storage-quotas

resolving merge conflicts (#9361)
IQSS · Sep 18, 2023 · a9036fc · a9036fc
2 parents 87c7383 + 4e1bc5b
commit a9036fc
Show file tree

Hide file tree

Showing 1,043 changed files with 31,341 additions and 18,335 deletions.
diff --git a/.env b/.env
@@ -1,4 +1,4 @@
 APP_IMAGE=gdcc/dataverse:unstable
 POSTGRES_VERSION=13
 DATAVERSE_DB_USER=dataverse
-SOLR_VERSION=8.11.1
+SOLR_VERSION=9.3.0
diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml
@@ -25,7 +25,7 @@ jobs:
                   ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge'
             - uses: actions/setup-java@v3
               with:
-                  java-version: "11"
+                  java-version: "17"
                   distribution: 'adopt'
             - uses: actions/cache@v3
               with:
@@ -57,27 +57,40 @@ jobs:
               run: |
                   echo "IMAGE_TAG=$(echo "${{ github.event.client_payload.pull_request.head.ref }}" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
 
-            - name: Deploy multi-arch application container image
-              run: mvn -Pct deploy -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
+            # Necessary to split as otherwise the submodules are not available (deploy skips install)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
+              run: >
+                  mvn -B -f modules/dataverse-parent
+                  -P ct -pl edu.harvard.iq:dataverse -am
+                  install
+            - name: Deploy multi-arch application and configbaker container image
+              run: >
+                  mvn 
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
+                  -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }}
+                  -Pct deploy
 
             - uses: marocchino/sticky-pull-request-comment@v2
               with:
-                  header: app-registry-push
+                  header: registry-push
                   hide_and_recreate: true
                   hide_classify: "OUTDATED"
                   number: ${{ github.event.client_payload.pull_request.number }}
                   message: |
-                      :package: Pushed preview application image as
+                      :package: Pushed preview images as
                       ```
                       ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
                       ```
-                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.
+                      ```
+                      ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
 
             # Leave a note when things have gone sideways
             - uses: peter-evans/create-or-update-comment@v3
               if: ${{ failure() }}
               with:
                   issue-number: ${{ github.event.client_payload.pull_request.number }}
                   body: >
-                      :package: Could not push preview image :disappointed:.
-                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.
+                      :package: Could not push preview images :disappointed:.
+                      See [log](https://github.com/IQSS/dataverse/actions/runs/${{ github.run_id }}) for details.
diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml
@@ -11,6 +11,7 @@ on:
             - master
         paths:
             - 'src/main/docker/**'
+            - 'modules/container-configbaker/**'
             - '.github/workflows/container_app_push.yml'
 
 env:
@@ -35,26 +36,24 @@ jobs:
             - name: Checkout repository
               uses: actions/checkout@v3
 
-            - name: Set up JDK 11
+            - name: Set up JDK
               uses: actions/setup-java@v3
               with:
-                  java-version: "11"
-                  distribution: 'adopt'
-            - name: Cache Maven packages
-              uses: actions/cache@v3
-              with:
-                  path: ~/.m2
-                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-                  restore-keys: ${{ runner.os }}-m2
+                  java-version: "17"
+                  distribution: temurin
+                  cache: maven
 
-            - name: Build app container image with local architecture
-              run: mvn -Pct package
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
+              run: >
+                  mvn -B -f modules/dataverse-parent
+                  -P ct -pl edu.harvard.iq:dataverse -am
+                  install
 
-            # TODO: add smoke / integration testing here
+            # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false")
 
     hub-description:
         needs: build
-        name: Push image description to Docker Hub
+        name: Push image descriptions to Docker Hub
         # Run this when triggered via push or schedule as reused workflow from base / maven unit tests.
         # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks.
         if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }}
@@ -68,6 +67,13 @@ jobs:
                   repository: gdcc/dataverse
                   short-description: "Dataverse Application Container Image providing the executable"
                   readme-filepath: ./src/main/docker/README.md
+            - uses: peter-evans/dockerhub-description@v3
+              with:
+                  username: ${{ secrets.DOCKERHUB_USERNAME }}
+                  password: ${{ secrets.DOCKERHUB_TOKEN }}
+                  repository: gdcc/configbaker
+                  short-description: "Dataverse Config Baker Container Image providing setup tooling and more"
+                  readme-filepath: ./modules/container-configbaker/README.md
 
     # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets.
     # We check for them here and subsequent jobs can rely on this to decide if they shall run.
@@ -93,22 +99,21 @@ jobs:
         name: "Package & Publish"
         runs-on: ubuntu-latest
         # Only run this job if we have access to secrets. This is true for events like push/schedule which run in
-        # context of main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
-        if: needs.check-secrets.outputs.available == 'true'
+        # context of the main repo, but for PRs only true if coming from the main repo! Forks have no secret access.
+        #
+        # Note: The team's decision was to not auto-deploy an image on any git push where no PR exists (yet).
+        #       Accordingly, only run for push events on branches develop and master.
+        if: needs.check-secrets.outputs.available == 'true' &&
+            ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name)))
         steps:
             - uses: actions/checkout@v3
             - uses: actions/setup-java@v3
               with:
-                  java-version: "11"
-                  distribution: 'adopt'
-            - uses: actions/cache@v3
-              with:
-                  path: ~/.m2
-                  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-                  restore-keys: ${{ runner.os }}-m2
+                  java-version: "17"
+                  distribution: temurin
 
             # Depending on context, we push to different targets. Login accordingly.
-            - if: ${{ github.event_name != 'pull_request' }}
+            - if: github.event_name != 'pull_request'
               name: Log in to Docker Hub registry
               uses: docker/login-action@v2
               with:
@@ -136,18 +141,31 @@ jobs:
                   echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV
                   echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV
 
-            - name: Deploy multi-arch application container image
-              run: mvn -Pct deploy -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
+            # Necessary to split as otherwise the submodules are not available (deploy skips install)
+            - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests)
+              run: >
+                  mvn -B -f modules/dataverse-parent
+                  -P ct -pl edu.harvard.iq:dataverse -am
+                  install
+            - name: Deploy multi-arch application and configbaker container image
+              run: >
+                  mvn 
+                  -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }}
+                  ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }}
+                  -P ct deploy
 
             - uses: marocchino/sticky-pull-request-comment@v2
               if: ${{ github.event_name == 'pull_request' }}
               with:
-                  header: app-registry-push
+                  header: registry-push
                   hide_and_recreate: true
                   hide_classify: "OUTDATED"
                   message: |
-                      :package: Pushed preview application image as
+                      :package: Pushed preview images as
                       ```
                       ghcr.io/gdcc/dataverse:${{ env.IMAGE_TAG }}
                       ```
-                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container/package/dataverse). Use by referencing with full name as printed above, mind the registry name.
+                      ```
+                      ghcr.io/gdcc/configbaker:${{ env.IMAGE_TAG }}
+                      ```
+                      :ship: [See on GHCR](https://github.com/orgs/gdcc/packages/container). Use by referencing with full name as printed above, mind the registry name.
diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml
@@ -34,7 +34,7 @@ jobs:
             packages: read
         strategy:
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
         # Only run in upstream repo - avoid unnecessary runs in forks
         if: ${{ github.repository_owner == 'IQSS' }}
 

diff --git a/.github/workflows/cypress_ui.yml.future b/.github/workflows/cypress_ui.yml.future
@@ -2,6 +2,7 @@
 #
 #   THIS IS AN OLD TRAVIS-CI.ORG JOB FILE
 #   To be used with Github Actions, it would be necessary to refactor it.
+#   In addition, it needs to be rewritten to use our modern containers.
 #   Keeping it as the future example it has been before.
 #   See also #5846
 #
@@ -30,8 +31,6 @@ jobs:
         directories:
           # we also need to cache folder with Cypress binary
           - ~/.cache
-          # we want to cache the Glassfish and Solr dependencies as well
-          - conf/docker-aio/dv/deps
       before_install:
         - cd tests
       install:

diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml
@@ -0,0 +1,85 @@
+name: 'Deploy to Beta Testing'
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'zulu'
+          java-version: '17'
+
+      - name: Enable API Session Auth feature flag
+        working-directory: src/main/resources/META-INF
+        run: echo -e "dataverse.feature.api-session-auth=true" >> microprofile-config.properties
+
+      - name: Set build number
+        run: scripts/installer/custom-build-number
+
+      - name: Build application war
+        run: mvn package
+
+      - name: Get war file name
+        working-directory: target
+        run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
+
+      - name: Upload war artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: built-app
+          path: ./target/${{ env.war_file }}
+
+  deploy-to-payara:
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download war artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: built-app
+          path: ./
+
+      - name: Get war file name
+        run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV
+
+      - name: Copy war file to remote instance
+        uses: appleboy/scp-action@master
+        with:
+          host: ${{ secrets.BETA_PAYARA_INSTANCE_HOST }}
+          username: ${{ secrets.BETA_PAYARA_INSTANCE_USERNAME }}
+          key: ${{ secrets.BETA_PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
+          source: './${{ env.war_file }}'
+          target: '/home/${{ secrets.BETA_PAYARA_INSTANCE_USERNAME }}'
+          overwrite: true
+
+      - name: Execute payara war deployment remotely
+        uses: appleboy/[email protected]
+        env:
+          INPUT_WAR_FILE: ${{ env.war_file }}
+        with:
+          host: ${{ secrets.BETA_PAYARA_INSTANCE_HOST }}
+          username: ${{ secrets.BETA_PAYARA_INSTANCE_USERNAME }}
+          key: ${{ secrets.BETA_PAYARA_INSTANCE_SSH_PRIVATE_KEY }}
+          envs: INPUT_WAR_FILE
+          script: |
+            APPLICATION_NAME=dataverse-backend
+            ASADMIN='/usr/local/payara6/bin/asadmin --user admin'
+            $ASADMIN undeploy $APPLICATION_NAME
+            $ASADMIN stop-domain
+            rm -rf /usr/local/payara6/glassfish/domains/domain1/generated
+            rm -rf /usr/local/payara6/glassfish/domains/domain1/osgi-cache
+            $ASADMIN start-domain
+            $ASADMIN deploy --name $APPLICATION_NAME $INPUT_WAR_FILE
+            $ASADMIN stop-domain
+            $ASADMIN start-domain
diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml
@@ -6,52 +6,67 @@ on:
             - "**.java"
             - "pom.xml"
             - "modules/**/pom.xml"
+            - "!modules/container-base/**"
+            - "!modules/dataverse-spi/**"
     pull_request:
         paths:
             - "**.java"
             - "pom.xml"
             - "modules/**/pom.xml"
+            - "!modules/container-base/**"
+            - "!modules/dataverse-spi/**"
 
 jobs:
     unittest:
         name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Unit Tests
         strategy:
             fail-fast: false
             matrix:
-                jdk: [ '11' ]
+                jdk: [ '17' ]
                 experimental: [false]
                 status:  ["Stable"]
-                #
-                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
-                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
-                # (This is a limitation of how Github is currently handling these things.)
-                #
-                #include:
-                #    - jdk: '17'
-                #      experimental: true
-                #      status: "Experimental"
         continue-on-error: ${{ matrix.experimental }}
         runs-on: ubuntu-latest
         steps:
-          - uses: actions/checkout@v2
+          - uses: actions/checkout@v3
           - name: Set up JDK ${{ matrix.jdk }}
-            uses: actions/setup-java@v2
+            uses: actions/setup-java@v3
             with:
                 java-version: ${{ matrix.jdk }}
-                distribution: 'adopt'
-          - name: Cache Maven packages
-            uses: actions/cache@v2
-            with:
-                path: ~/.m2
-                key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-                restore-keys: ${{ runner.os }}-m2
+                distribution: temurin
+                cache: maven
+
+          # The reason why we use "install" here is that we want the submodules to be available in the next step.
+          # Also, we can cache them this way for jobs triggered by this one.
           - name: Build with Maven
-            run: mvn -DcompilerArgument=-Xlint:unchecked -Dtarget.java.version=${{ matrix.jdk }} -P all-unit-tests clean test
+            run: > 
+                mvn -B -f modules/dataverse-parent
+                -Dtarget.java.version=${{ matrix.jdk }}
+                -DcompilerArgument=-Xlint:unchecked -P all-unit-tests
+                -pl edu.harvard.iq:dataverse -am
+                install
+
           - name: Maven Code Coverage
             env:
                 CI_NAME: github
                 COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }}
-            run: mvn -V -B jacoco:report coveralls:report -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
+            # The coverage commit is sometimes flaky. Don't bail out just because this optional step failed.
+            continue-on-error: true
+            run: >
+                mvn -B
+                -DrepoToken=${COVERALLS_SECRET} -DpullRequest=${{ github.event.number }}
+                jacoco:report coveralls:report
+
+          # We don't want to cache the WAR file, so delete it
+          - run: rm -rf ~/.m2/repository/edu/harvard/iq/dataverse
+
+          # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub.
+          # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads
+          - uses: actions/upload-artifact@v3
+            with:
+                name: dataverse-java${{ matrix.jdk }}.war
+                path: target/dataverse*.war
+                retention-days: 7
     push-app-img:
         name: Publish App Image
         permissions: