Merge remote-tracking branch 'upstream/main' into w2p-111756_find-sit…

…e-performance-improvement-7.6

 Conflicts:
	dspace-api/src/main/java/org/dspace/content/dao/impl/SiteDAOImpl.java

.codecov.yml

@@ -4,13 +4,6 @@
 # Can be validated via instructions at:
 # https://docs.codecov.io/docs/codecov-yaml#validate-your-repository-yaml
 
-# Tell Codecov not to send a coverage notification until (at least) 2 builds are completed
-# Since we run Unit & Integration tests in parallel, this lets Codecov know that coverage
-# needs to be merged across those builds
-codecov:
-  notify:
-    after_n_builds: 2
-
 # Settings related to code coverage analysis
 coverage:
   status:

.dockerignore

@@ -6,6 +6,5 @@ dspace/modules/*/target/
 Dockerfile.*
 dspace/src/main/docker/dspace-postgres-pgcrypto
 dspace/src/main/docker/dspace-postgres-pgcrypto-curl
-dspace/src/main/docker/solr
 dspace/src/main/docker/README.md
 dspace/src/main/docker-compose/

.github/pull_request_template.md

@@ -1,7 +1,7 @@
 ## References
 _Add references/links to any related issues or PRs. These may include:_
-* Fixes #[issue-number]
-* Related to [REST Contract](https://github.com/DSpace/Rest7Contract)
+* Fixes #`issue-number` (if this fixes an issue ticket)
+* Related to DSpace/RestContract#`pr-number`  (if a corresponding REST Contract PR exists)
 
 ## Description
 Short summary of changes (1-2 sentences).
@@ -22,5 +22,7 @@ _This checklist provides a reminder of what we are going to look for when review
 - [ ] My PR passes Checkstyle validation based on the [Code Style Guide](https://wiki.lyrasis.org/display/DSPACE/Code+Style+Guide).
 - [ ] My PR includes Javadoc for _all new (or modified) public methods and classes_. It also includes Javadoc for large or complex private methods.
 - [ ] My PR passes all tests and includes new/updated Unit or Integration Tests based on the [Code Testing Guide](https://wiki.lyrasis.org/display/DSPACE/Code+Testing+Guide).
-- [ ] If my PR includes new, third-party dependencies (in any `pom.xml`), I've made sure their licenses align with the [DSpace BSD License](https://github.com/DSpace/DSpace/blob/main/LICENSE) based on the [Licensing of Contributions](https://wiki.lyrasis.org/display/DSPACE/Code+Contribution+Guidelines#CodeContributionGuidelines-LicensingofContributions) documentation.
-- [ ] If my PR modifies the REST API, I've linked to the REST Contract page (or open PR) related to this change.
+- [ ] If my PR includes new libraries/dependencies (in any `pom.xml`), I've made sure their licenses align with the [DSpace BSD License](https://github.com/DSpace/DSpace/blob/main/LICENSE) based on the [Licensing of Contributions](https://wiki.lyrasis.org/display/DSPACE/Code+Contribution+Guidelines#CodeContributionGuidelines-LicensingofContributions) documentation.
+- [ ] If my PR modifies REST API endpoints, I've opened a separate [REST Contract](https://github.com/DSpace/RestContract/blob/main/README.md) PR related to this change.
+- [ ] If my PR includes new configurations, I've provided basic technical documentation in the PR itself.
+- [ ] If my PR fixes an issue ticket, I've [linked them together](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).

.github/workflows/build.yml

@@ -6,72 +6,106 @@ name: Build
 # Run this Build for all pushes / PRs to current branch
 on: [push, pull_request]
 
+permissions:
+  contents: read  #  to fetch code (actions/checkout)
+
 jobs:
   tests:
     runs-on: ubuntu-latest
     env:
       # Give Maven 1GB of memory to work with
-      # Suppress all Maven "downloading" messages in Travis logs (see https://stackoverflow.com/a/35653426)
-      # This also slightly speeds builds, as there is less logging
-      MAVEN_OPTS: "-Xmx1024M -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
+      MAVEN_OPTS: "-Xmx1024M"
     strategy:
       # Create a matrix of two separate configurations for Unit vs Integration Tests
       # This will ensure those tasks are run in parallel
+      # Also specify version of Java to use (this can allow us to optionally run tests on multiple JDKs in future)
       matrix:
         include:
-          # NOTE: Unit Tests include deprecated REST API v6 (as it has unit tests)
+          # NOTE: Unit Tests include a retry for occasionally failing tests
+          #  - surefire.rerunFailingTestsCount => try again for flakey tests, and keep track of/report on number of retries
           - type: "Unit Tests"
-            mvnflags: "-DskipUnitTests=false -Pdspace-rest"
+            java: 11
+            mvnflags: "-DskipUnitTests=false -Dsurefire.rerunFailingTestsCount=2"
             resultsdir: "**/target/surefire-reports/**"
           # NOTE: ITs skip all code validation checks, as they are already done by Unit Test job.
           #  - enforcer.skip     => Skip maven-enforcer-plugin rules
           #  - checkstyle.skip   => Skip all checkstyle checks by maven-checkstyle-plugin
           #  - license.skip      => Skip all license header checks by license-maven-plugin
           #  - xml.skip          => Skip all XML/XSLT validation by xml-maven-plugin
+          #  - failsafe.rerunFailingTestsCount => try again for flakey tests, and keep track of/report on number of retries
           - type: "Integration Tests"
-            mvnflags: "-DskipIntegrationTests=false -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true"
+            java: 11
+            mvnflags: "-DskipIntegrationTests=false -Denforcer.skip=true -Dcheckstyle.skip=true -Dlicense.skip=true -Dxml.skip=true -Dfailsafe.rerunFailingTestsCount=2"
             resultsdir: "**/target/failsafe-reports/**"
       # Do NOT exit immediately if one matrix job fails
       # This ensures ITs continue running even if Unit Tests fail, or visa versa
       fail-fast: false
+    name: Run ${{ matrix.type }}
     # These are the actual CI steps to perform per job
     steps:
       # https://github.com/actions/checkout
       - name: Checkout codebase
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
       # https://github.com/actions/setup-java
-      - name: Install JDK 11
-        uses: actions/setup-java@v1
-        with:
-          java-version: 11
-
-      # https://github.com/actions/cache
-      - name: Cache Maven dependencies
-        uses: actions/cache@v2
+      - name: Install JDK ${{ matrix.java }}
+        uses: actions/setup-java@v3
         with:
-          # Cache entire ~/.m2/repository
-          path: ~/.m2/repository
-          # Cache key is hash of all pom.xml files. Therefore any changes to POMs will invalidate cache
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-maven-
+          java-version: ${{ matrix.java }}
+          distribution: 'temurin'
+          cache: 'maven'
 
       # Run parallel Maven builds based on the above 'strategy.matrix'
       - name: Run Maven ${{ matrix.type }}
         env:
           TEST_FLAGS: ${{ matrix.mvnflags }}
-        run: mvn install -B -V -P-assembly -Pcoverage-report $TEST_FLAGS
+        run: mvn --no-transfer-progress -V install -P-assembly -Pcoverage-report $TEST_FLAGS
 
       # If previous step failed, save results of tests to downloadable artifact for this job
       # (This artifact is downloadable at the bottom of any job's summary page)
       - name: Upload Results of ${{ matrix.type }} to Artifact
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.type }} results
           path: ${{ matrix.resultsdir }}
-          retention-days: 7
 
-      # https://github.com/codecov/codecov-action
+      # Upload code coverage report to artifact, so that it can be shared with the 'codecov' job (see below)
+      - name: Upload code coverage report to Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.type }} coverage report
+          path: 'dspace/target/site/jacoco-aggregate/jacoco.xml'
+          retention-days: 14
+
+  # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test
+  # job above. This is necessary because Codecov uploads seem to randomly fail at times.
+  # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954
+  codecov:
+    # Must run after 'tests' job above
+    needs: tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Download artifacts from previous 'tests' job
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v3
+
+      # Now attempt upload to Codecov using its action.
+      # NOTE: We use a retry action to retry the Codecov upload if it fails the first time.
+      #
+      # Retry action: https://github.com/marketplace/actions/retry-action
+      # Codecov action: https://github.com/codecov/codecov-action
       - name: Upload coverage to Codecov.io
-        uses: codecov/codecov-action@v1
+        uses: Wandalen/[email protected]
+        with:
+          action: codecov/codecov-action@v3
+          # Ensure codecov-action throws an error when it fails to upload
+          with: |
+            fail_ci_if_error: true
+          # Try re-running action 5 times max
+          attempt_limit: 5
+          # Run again in 30 seconds
+          attempt_delay: 30000

.github/workflows/codescan.yml

@@ -0,0 +1,63 @@
+# DSpace CodeQL code scanning configuration for GitHub
+# https://docs.github.com/en/code-security/code-scanning
+#
+# NOTE: Code scanning must be run separate from our default build.yml
+# because CodeQL requires a fresh build with all tests *disabled*.
+name: "Code Scanning"
+
+# Run this code scan for all pushes / PRs to main or maintenance branches. Also run once a week.
+on:
+  push:
+    branches:
+      - main
+      - 'dspace-**'
+  pull_request:
+    branches:
+      - main
+      - 'dspace-**'
+    # Don't run if PR is only updating static documentation
+    paths-ignore:
+      - '**/*.md'
+      - '**/*.txt'
+  schedule:
+    - cron: "37 0 * * 1"
+
+jobs:
+  analyze:
+    name: Analyze Code
+    runs-on: ubuntu-latest
+    # Limit permissions of this GitHub action. Can only write to security-events
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      # https://github.com/actions/checkout
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # https://github.com/actions/setup-java
+      - name: Install JDK
+        uses: actions/setup-java@v3
+        with:
+          java-version: 11
+          distribution: 'temurin'
+
+      # Initializes the CodeQL tools for scanning.
+      # https://github.com/github/codeql-action
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          # Codescan Javascript as well since a few JS files exist in REST API's interface
+          languages: java, javascript
+
+      # Autobuild attempts to build any compiled languages
+      # NOTE: Based on testing, this autobuild process works well for DSpace. A custom
+      # DSpace build w/caching (like in build.yml) was about the same speed as autobuild.
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # Perform GitHub Code Scanning.
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2