Merge branch 'eclipse-sw360:main' into master

siemens · Apr 6, 2023 · 1f4869d · 1f4869d
2 parents b03b3bc + 0768cd6
commit 1f4869d
Show file tree

Hide file tree

Showing 54 changed files with 1,130 additions and 257 deletions.
diff --git a/.github/testForLicenseHeaders.sh b/.github/testForLicenseHeaders.sh
@@ -38,6 +38,7 @@ done <<< "$(git ls-files \
     | grep -v .pre-commit-config.yaml \
     | grep -v 'id_rsa' \
     | grep -v '.versions' \
+    | grep -v 'default_secrets' \
     | grep -Ev '*/asciidoc/*')"
 
 if [ "$failure" = true ]; then

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -29,6 +29,7 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+    - uses: webiny/[email protected]
 
     - name: Verify license headers
       run: |
@@ -68,7 +69,9 @@ jobs:
         path: |
           /usr/local/bin/thrift
           /usr/share/thrift/${{ env.THRIFT_VERSION }}
-        key: ${{ runner.os }}-thriftbin
+        key: ${{ runner.os }}-thrift-${{ hashFiles('/usr/local/bin/thrift') }}
+        restore-keys: |
+          ${{ runner.os }}-thrift-
 
     - name: Install Thrift
       if: steps.cache-thrift.outputs.cache-hit != 'true'
@@ -86,15 +89,15 @@ jobs:
 
     - name: Build SW360
       run: |
-        mvn clean package --no-transfer-progress -P deploy -Dhelp-docs=true -Dbase.deploy.dir=. -Dliferay.deploy.dir=${PWD}/deploy -Dbackend.deploy.dir=${PWD}/deploy/webapps -Drest.deploy.dir=${PWD}/deploy/webapps -DRunComponentVisibilityRestrictionTest=false -DRunPrivateProjectAccessTest=false
+        mvn clean package --no-transfer-progress -P deploy -Dhelp-docs=true -Dbase.deploy.dir=. -Dliferay.deploy.dir=${PWD}/deploy -Dbackend.deploy.dir=${PWD}/deploy/webapps -Drest.deploy.dir=${PWD}/deploy/webapps -DRunComponentVisibilityRestrictionTest=false -DRunPrivateProjectAccessTest=false -DRunRestForceUpdateTest=false
 
     - name: Run PrivateProjectAccessTest
       run: |
         cd build-configuration
         mvn install
         cd ..
         cd libraries/datahandler
-        mvn test -Dtest=ProjectPermissionsVisibilityTest -DRunPrivateProjectAccessTest=true
+        mvn test -Dtest=ProjectPermissionsVisibilityTest -DRunPrivateProjectAccessTest=true -DRunRestForceUpdateTest=true
 
     - name: Deploy Backend and Rest Server
       run: |
@@ -109,4 +112,3 @@ jobs:
       run: |
         cd clients
         mvn clean install --no-transfer-progress -DRunRestIntegrationTest=true
-
diff --git a/.github/workflows/docker_deploy.yml b/.github/workflows/docker_deploy.yml
@@ -13,18 +13,20 @@
 name: Docker Build
 
 on:
-  push:
-    schedule:
+  schedule:
       - cron: '0 0 * * *' # Midnight
+  push:
     tags:
       - 'sw360-*'
     paths-ignore:
       - "**.md"
+
 env:
   REGISTRY: ghcr.io
 
 jobs:
   docker_push:
+    if: ${{ github.event.schedule }} == '0 0 * * *' || ${{ github.event.act }}
     name: Build Docker Image
     runs-on: ubuntu-22.04
     permissions:
@@ -64,7 +66,7 @@ jobs:
         run: echo "Will be tagged as ${{ steps.meta_base.outputs.tags }} and labeled as ${{ steps.meta_base.outputs.labels }}"
 
       - name: Build sw360 base container
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           target: base
@@ -94,7 +96,7 @@ jobs:
         run: echo "Will be tagged as ${{ steps.meta_thrift.outputs.tags }} and labeled as ${{ steps.meta_thrift.outputs.labels }}"
 
       - name: Build sw360 Thrift container
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           target: sw360thrift
@@ -111,7 +113,7 @@ jobs:
           cache-to: type=gha,scope=thrift,mode=max
 
       #------------------------------------------------
-      # CLucene
+      # ClLucene
       - name: Extract components metadata (tags, labels) for clucene image
         id: meta_clucene
         uses: docker/metadata-action@v4
@@ -123,15 +125,14 @@ jobs:
         run: echo "Will be tagged as ${{ steps.meta_clucene.outputs.tags }} and labeled as ${{ steps.meta_clucene.outputs.labels }}"
 
       - name: Build sw360 clucene container
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           target: sw360clucene
           push: true
           load: false
           build-args: |
             CLUCENE_VERSION=${{ env.CLUCENE_VERSION }}
-            MAVEN_VERSION=${{ env.MAVEN_VERSION }}
           tags: |
             ${{ steps.meta_clucene.outputs.tags }}
             ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/clucene:${{ env.CLUCENE_VERSION }}
@@ -153,14 +154,14 @@ jobs:
         run: echo "Will be tagged as ${{ steps.meta_sw360.outputs.tags }} and labeled as ${{ steps.meta_sw360.outputs.labels }}"
 
       - name: Build sw360 build container
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           target: sw360
           push: true
           load: false
-          build-args: |
-            MAVEN_VERSION=${{ env.MAVEN_VERSION }}
+          secret-files: |
+            "sw360=./scripts/docker-config/default_secrets"
           tags: |
             ${{ steps.meta_sw360.outputs.tags }}
             ${{ env.REGISTRY }}/${{ env.ORG_BASE_NAME }}/binaries:${{ env.GIT_REVISION }}
@@ -174,7 +175,7 @@ jobs:
 
       #------------------------------------------------
       # sw360 runtime container
-      - name: Extract components metadata (tags, labels) for clucene image
+      - name: Extract components metadata (tags, labels) runtime image
         id: meta_runtime
         uses: docker/metadata-action@v4
         with:
@@ -185,7 +186,7 @@ jobs:
         run: echo "Will be tagged as ${{ steps.meta_runtime.outputs.tags }} and labeled as ${{ steps.meta_runtime.outputs.labels }}"
 
       - name: Build sw360 build container
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           target: runtime

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,20 @@
 repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+    -   id: check-merge-conflict
+    # -   id: end-of-file-fixer
+    # -   id: trailing-whitespace
+
+-   repo: https://github.com/compilerla/conventional-pre-commit
+    rev: 'v2.1.1'
+    hooks:
+    -   id: conventional-pre-commit
+        stages: [commit-msg]
+
 -   repo: https://github.com/ejba/pre-commit-maven
     rev: v0.3.3
     hooks:
     -   id: maven-spotless-apply
+        stages: [manual]
+        always_run: false
diff --git a/.versions b/.versions
@@ -1,5 +1,4 @@
 CLUCENE_VERSION=2.1.0
 THRIFT_VERSION=0.16.0
-MAVEN_VERSION=3.8.7
 LIFERAY_VERSION=7.4.3.18-ga18
 LIFERAY_SOURCE=liferay-ce-portal-tomcat-7.4.3.18-ga18-20220329092001364.tar.gz
diff --git a/Dockerfile b/Dockerfile
@@ -33,8 +33,7 @@ ARG HOMEDIR=/workspace
 ENV HOME=$HOMEDIR
 
 # Base system
-RUN --mount=type=cache,mode=0755,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,mode=0755,target=/var/lib/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt \
     apt-get update -qq \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     ca-certificates \
@@ -85,8 +84,7 @@ FROM ubuntu:jammy AS sw360thriftbuild
 ARG BASEDIR="/build"
 ARG THRIFT_VERSION
 
-RUN --mount=type=cache,mode=0755,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,mode=0755,target=/var/lib/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt \
     apt-get -qq update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     bison \
@@ -109,15 +107,13 @@ COPY --from=sw360thriftbuild /usr/local/bin/thrift /usr/local/bin/thrift
 
 #--------------------------------------------------------------------------------------------------
 # Couchdb-Lucene
-FROM eclipse-temurin:11-jdk-jammy as sw360clucenebuild
+FROM maven:3.9-eclipse-temurin-11 as sw360clucenebuild
 
 ARG CLUCENE_VERSION
-ARG MAVEN_VERSION
 
 WORKDIR /build
 
-RUN --mount=type=cache,mode=0755,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,mode=0755,target=/var/lib/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt \
     apt-get -qq update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     gettext-base \
@@ -126,7 +122,6 @@ RUN --mount=type=cache,mode=0755,target=/var/cache/apt,sharing=locked \
 # Prepare maven from binary to avoid wrong java dependencies and proxy
 COPY scripts/docker-config/mvn-proxy-settings.xml /etc
 COPY scripts/docker-config/set_proxy.sh /usr/local/bin/setup_maven_proxy
-RUN curl -JL https://dlcdn.apache.org/maven/maven-3/"$MAVEN_VERSION"/binaries/apache-maven-"$MAVEN_VERSION"-bin.tar.gz | tar -xz --strip-components=1 -C /usr/local
 RUN chmod a+x /usr/local/bin/setup_maven_proxy \
     && setup_maven_proxy
 
@@ -136,7 +131,7 @@ COPY ./scripts/patches/couchdb-lucene.patch /var/tmp/couchdb-lucene.patch
 
 # Build CLucene
 RUN --mount=type=tmpfs,target=/build \
-    --mount=type=cache,mode=0755,target=/root/.m2,rw,sharing=locked \
+    --mount=type=cache,target=/root/.m2 \
     curl -JL https://github.com/rnewson/couchdb-lucene/archive/v"$CLUCENE_VERSION".tar.gz | tar -C /build -xz --strip-components=1 \
     && patch -p1 < /var/tmp/couchdb-lucene.patch \
     && cp /var/tmp/couchdb-lucene.ini src/main/resources/couchdb-lucene.ini \
@@ -152,15 +147,14 @@ COPY --from=sw360clucenebuild /couchdb-lucene.war /couchdb-lucene.war
 # So when decide to use as development, only this last stage
 # is triggered by buildkit images
 
-FROM eclipse-temurin:11-jdk-jammy as sw360build
-
-ARG MAVEN_VERSION
+FROM maven:3.9-eclipse-temurin-11 as sw360build
 
 WORKDIR /build
 
+SHELL ["/bin/bash", "-c"]
+
 # Install mkdocs to generate documentation
-RUN --mount=type=cache,mode=0755,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,mode=0755,target=/var/lib/apt,sharing=locked \
+RUN --mount=type=cache,target=/var/cache/apt \
     apt-get update -qq \
     && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \
     gettext-base \
@@ -175,15 +169,21 @@ RUN --mount=type=cache,mode=0755,target=/var/cache/apt,sharing=locked \
 # Prepare maven from binary to avoid wrong java dependencies and proxy
 COPY scripts/docker-config/mvn-proxy-settings.xml /etc
 COPY scripts/docker-config/set_proxy.sh /usr/local/bin/setup_maven_proxy
-RUN curl -JL https://dlcdn.apache.org/maven/maven-3/"$MAVEN_VERSION"/binaries/apache-maven-"$MAVEN_VERSION"-bin.tar.gz | tar -xz --strip-components=1 -C /usr/local
 RUN chmod a+x /usr/local/bin/setup_maven_proxy \
     && setup_maven_proxy
 
 COPY --from=sw360thrift /usr/local/bin/thrift /usr/bin
 
 RUN --mount=type=bind,target=/build/sw360,rw \
-    --mount=type=cache,mode=0755,target=/root/.m2,rw,sharing=locked \
+    --mount=type=cache,target=/root/.m2 \
+    --mount=type=secret,id=sw360 \
     cd /build/sw360 \
+    && set -a \
+    && source /run/secrets/sw360 \
+    && envsubst < scripts/docker-config/couchdb.properties.template | tee scripts/docker-config/etc_sw360/couchdb.properties \
+    && set +a \
+    && cp scripts/docker-config/etc_sw360/couchdb.properties build-configuration/resources/ \
+    && cp -a scripts/docker-config/etc_sw360 /etc/sw360 \
     && mvn clean package \
     -P deploy \
     -Dtest=org.eclipse.sw360.rest.resourceserver.restdocs.* \
@@ -205,6 +205,7 @@ RUN bash /bin/slim.sh
 FROM scratch AS sw360
 COPY --from=sw360build /sw360_deploy /sw360_deploy
 COPY --from=sw360build /sw360_tomcat_webapps /sw360_tomcat_webapps
+COPY --from=sw360build /etc/sw360 /etc/sw360
 
 #--------------------------------------------------------------------------------------------------
 # Runtime image
@@ -222,14 +223,15 @@ COPY --chown=$USERNAME:$USERNAME --from=sw360 /sw360_tomcat_webapps/slim-wars/*.
 COPY --chown=$USERNAME:$USERNAME --from=sw360 /sw360_tomcat_webapps/*.jar /app/sw360/tomcat/webapps/
 # Shared streamlined jar libs
 COPY --chown=$USERNAME:$USERNAME --from=sw360 /sw360_tomcat_webapps/libs/*.jar /app/sw360/tomcat/shared/
+# Modified etc
+COPY --chown=$USERNAME:$USERNAME --from=sw360 /etc/sw360 /etc/sw360
 
 # Make catalina understand shared directory
 RUN dos2unix /app/sw360/tomcat/conf/catalina.properties \
     && sed -i "s,shared.loader=,shared.loader=/app/sw360/tomcat/shared/*.jar,g" /app/sw360/tomcat/conf/catalina.properties
 
 # Copy liferay/sw360 config files
 COPY --chown=$USERNAME:$USERNAME ./scripts/docker-config/portal-ext.properties /app/sw360/portal-ext.properties
-COPY --chown=$USERNAME:$USERNAME ./scripts/docker-config/etc_sw360 /etc/sw360
 COPY --chown=$USERNAME:$USERNAME ./scripts/docker-config/entry_point.sh /app/entry_point.sh
 
 STOPSIGNAL SIGINT

diff --git a/README_DOCKER.md b/README_DOCKER.md
@@ -23,21 +23,20 @@
     The script will build multiple intermediary images.
     Subsequent builds will only build the differences
 
-    For runtime, docker compose for sw360 are configured with default entries on docker-compose.yml.
+    To configure couchdb, create a file containing the necessary credentials.
 
-    The config entries that can be modifiled:
+    A template of this file can be found in:
+    `scripts/docker_config/default_secrets`
 
+    Example:
     ```ini
-    # Postgres
-    POSTGRES_USER=liferay
-    POSTGRES_PASSWORD=liferay
-    POSTGRES_DB=lportal
-    # Couchdb
+    COUCHDB_URL=http://couchdb:5984
     COUCHDB_USER=admin
     COUCHDB_PASSWORD=password
-    COUCHDB_CREATE_DATABASE=yes
     ```
 
+    To pass your file during build export a variable called **SECRETS** pointing to your file
+
 * Proxy during build stage
 
     Docker will detect if you configured proxy environment variables.
@@ -46,7 +45,7 @@
 
     * systemd based
       If you are using a regular systemd based docker:
-      * Create the following file **http_proxy.conf** on the directory **/etc/systemd/system/docker.service.d/**
+      * Create the following file **http_proxy.conf** on the directory `/etc/systemd/system/docker.service.d/`
 
       ```ini
       [Service]

diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/common/utils/BackendUtils.java b/backend/src-common/src/main/java/org/eclipse/sw360/common/utils/BackendUtils.java
@@ -18,10 +18,13 @@ public class BackendUtils {
     private static final String PROPERTIES_FILE_PATH = "/sw360.properties";
     protected static final Properties loadedProperties;
     public static final Boolean MAINLINE_STATE_ENABLED_FOR_USER;
+    public static final Boolean IS_FORCE_UPDATE_ENABLED;
 
     static {
         loadedProperties = CommonUtils.loadProperties(BackendUtils.class, PROPERTIES_FILE_PATH);
         MAINLINE_STATE_ENABLED_FOR_USER = Boolean.parseBoolean(loadedProperties.getProperty("mainline.state.enabled.for.user", "false"));
+        IS_FORCE_UPDATE_ENABLED = Boolean.parseBoolean(
+                System.getProperty("RunRestForceUpdateTest", loadedProperties.getProperty("rest.force.update.enabled", "false")));
     }
 
     protected BackendUtils() {