Merge pull request #141 from usdot-jpo-ode/release/2025-q1

Release/2025 q1
usdot-jpo-ode · Jan 31, 2025 · f287adb · f287adb
2 parents eb19125 + c498442
commit f287adb
Show file tree

Hide file tree

Showing 313 changed files with 9,364 additions and 7,596 deletions.
diff --git a/.github/workflows/artifact-publish.yml b/.github/workflows/artifact-publish.yml
@@ -0,0 +1,47 @@
+name: Publish Java Package
+
+on:
+  push:
+    tags:
+      - 'jpo-conflictmonitor-*'
+
+
+jobs:
+  conflictmonitor-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up JDK 21a
+      uses: actions/setup-java@v4
+      with:
+        java-version: '21'
+        distribution: 'temurin'
+        cache: 'maven'
+
+    - name: Remove snapshot from version
+      run: |
+        cd jpo-conflictmonitor
+        mvn versions:set -DremoveSnapshot
+
+    - name: Build with Maven
+      env:
+        SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        MAVEN_GITHUB_TOKEN_NAME: ${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+        MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+        MAVEN_GITHUB_ORG: ${{ github.repository_owner }}
+      run: |
+        cd jpo-conflictmonitor
+        mvn -s ../settings.xml -B package --file pom.xml -DskipTests -Ppackage-jar
+
+    - name: Publish to GitHub Packages
+      run: |
+        cd jpo-conflictmonitor
+        mvn -s ../settings.xml --batch-mode -Dgithub_organization=${{ github.repository_owner }} deploy -DskipTests -Ppackage-jar
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        MAVEN_GITHUB_TOKEN_NAME: ${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+        MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+        MAVEN_GITHUB_ORG: ${{ github.repository_owner }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,15 @@ jobs:
         uses: docker/setup-buildx-action@v2
       - name: Build
         uses: docker/build-push-action@v3
+        with:
+          build-args: |
+            MAVEN_GITHUB_TOKEN_NAME=${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+            MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }}
+            MAVEN_GITHUB_ORG=${{ github.repository_owner }}
+          secrets: |
+            MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
   sonar:
     runs-on: ubuntu-latest
@@ -25,15 +34,13 @@ jobs:
         with:
           java-version: "21"
           distribution: "temurin"
+          cache: 'maven'
       - name: Run Sonar
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          MAVEN_GITHUB_TOKEN_NAME: ${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+          MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          MAVEN_GITHUB_ORG: ${{ github.repository_owner }}
         run: |
-          cd jpo-geojsonconverter/jpo-ode
-          mvn install -DskipTests
-          cd ../jpo-geojsonconverter
-          mvn install -DskipTests
-          cd $GITHUB_WORKSPACE/test-message-sender
-          mvn -e -X clean install
-          cd $GITHUB_WORKSPACE/jpo-conflictmonitor
-          mvn -e -X clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar -Dsonar.projectKey=usdot-jpo-ode_jpo-conflictmonitor -Dsonar.projectName=jpo-conflictmonitor -Dsonar.organization=usdot-jpo-ode -Dsonar.host.url=https://sonarcloud.io -Dsonar.branch.name=$GITHUB_REF_NAME
+          cd jpo-conflictmonitor/
+          mvn -s ../settings.xml -e -X clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar -Dsonar.projectKey=usdot-jpo-ode_jpo-conflictmonitor -Dsonar.projectName=jpo-conflictmonitor -Dsonar.organization=usdot-jpo-ode -Dsonar.host.url=https://sonarcloud.io -Dsonar.branch.name=$GITHUB_REF_NAME
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -13,4 +13,13 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v3
+        with:
+          build-args: |
+            MAVEN_GITHUB_TOKEN_NAME=${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+            MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }}
+            MAVEN_GITHUB_ORG=${{ github.repository_owner }}
+          secrets: |
+            MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
@@ -20,7 +20,7 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Replcae Docker tag
+      - name: Replace Docker tag
         id: set_tag
         run: echo "TAG=$(echo ${GITHUB_REF##*/} | sed 's/\//-/g')" >> $GITHUB_ENV
 
@@ -29,3 +29,11 @@ jobs:
         with:
           push: true
           tags: usdotjpoode/jpo-conflictmonitor:${{ env.TAG }}
+          build-args: |
+            MAVEN_GITHUB_TOKEN_NAME=${{ vars.MAVEN_GITHUB_TOKEN_NAME }}
+            MAVEN_GITHUB_TOKEN=${{ secrets.MAVEN_GITHUB_TOKEN }}
+            MAVEN_GITHUB_ORG=${{ github.repository_owner }}
+          secrets: |
+            MAVEN_GITHUB_TOKEN: ${{ secrets.MAVEN_GITHUB_TOKEN }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,5 @@
 jpo-conflictmonitor/target/**
 jpo-conflictmonitor/bin/**
-jpo-deduplicator/target/**
-jpo-deduplicator/bin/**
 .env
 *.log
 .idea/

diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,3 @@
-[submodule "jpo-geojsonconverter"]
-	path = jpo-geojsonconverter
-	url = https://github.com/usdot-jpo-ode/jpo-geojsonconverter
+[submodule "jpo-utils"]
+	path = jpo-utils
+	url = https://github.com/usdot-jpo-ode/jpo-utils
diff --git a/Dedup_Dockerfile b/Dedup_Dockerfile
diff --git a/Dockerfile b/Dockerfile
@@ -1,35 +1,23 @@
-FROM maven:3.8-eclipse-temurin-21-alpine as builder
+FROM maven:3.8-eclipse-temurin-21-alpine AS builder
 
 WORKDIR /home
 
-# Copy only the files needed to avoid putting all sorts of junk from your local env on to the image
-COPY ./jpo-geojsonconverter/jpo-ode/pom.xml ./jpo-ode/
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-common/pom.xml ./jpo-ode/jpo-ode-common/
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-common/src ./jpo-ode/jpo-ode-common/src
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-plugins/pom.xml ./jpo-ode/jpo-ode-plugins/
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-plugins/src ./jpo-ode/jpo-ode-plugins/src
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-core/pom.xml ./jpo-ode/jpo-ode-core/
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-core/src ./jpo-ode/jpo-ode-core/src/
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-svcs/pom.xml ./jpo-ode/jpo-ode-svcs/
-COPY ./jpo-geojsonconverter/jpo-ode/jpo-ode-svcs/src ./jpo-ode/jpo-ode-svcs/src
+ARG MAVEN_GITHUB_TOKEN
+ARG MAVEN_GITHUB_ORG
 
-COPY ./jpo-geojsonconverter/jpo-geojsonconverter/pom.xml ./jpo-geojsonconverter/
-COPY ./jpo-geojsonconverter/jpo-geojsonconverter/src ./jpo-geojsonconverter/src
+ENV MAVEN_GITHUB_TOKEN=$MAVEN_GITHUB_TOKEN
+ENV MAVEN_GITHUB_ORG=$MAVEN_GITHUB_ORG
 
 COPY ./jpo-conflictmonitor/pom.xml ./jpo-conflictmonitor/
-COPY ./jpo-conflictmonitor/src ./jpo-conflictmonitor/src
-
-WORKDIR /home/jpo-ode
-
-RUN mvn install -DskipTests
-
-WORKDIR /home/jpo-geojsonconverter
-
-RUN mvn clean install -DskipTests
+COPY ./settings.xml ./jpo-conflictmonitor/
 
+# Download dependencies alone to cache them first
 WORKDIR /home/jpo-conflictmonitor
+RUN mvn -s settings.xml dependency:resolve
 
-RUN mvn clean package -DskipTests
+# Copy the source code and build the conflict monitor
+COPY ./jpo-conflictmonitor/src ./src
+RUN mvn -s settings.xml install -DskipTests
 
 FROM amazoncorretto:21
 
@@ -41,16 +29,47 @@ COPY --from=builder /home/jpo-conflictmonitor/target/jpo-conflictmonitor.jar /ho
 
 
 
+# Use jemalloc for RocksDB per Confluent recommendation:
+# https://docs.confluent.io/platform/current/streams/developer-guide/memory-mgmt.html#rocksdb
+RUN amazon-linux-extras install -y epel && \
+     yum install -y jemalloc-devel
+ENV LD_PRELOAD="/usr/lib64/libjemalloc.so"
+
+# Entrypoint for prod: JMX not exposed.
+# GC settings similar to Kafka recommendations, see: https://kafka.apache.org/documentation.html#java
+# Set max Java heap usage as percentage of total available memory.
 ENTRYPOINT ["java", \
-	"-Djava.rmi.server.hostname=$DOCKER_HOST_IP", \
-	"-Dcom.sun.management.jmxremote.port=9090", \
-	"-Dcom.sun.management.jmxremote.rmi.port=9090", \
-	"-Dcom.sun.management.jmxremote", \
-	"-Dcom.sun.management.jmxremote.local.only=true", \
-	"-Dcom.sun.management.jmxremote.authenticate=false", \
-	"-Dcom.sun.management.jmxremote.ssl=false", \
 	"-Dlogback.configurationFile=/home/logback.xml", \
+    "-XX:+UseG1GC", \
+    "-XX:MaxGCPauseMillis=20", \
+    "-XX:InitiatingHeapOccupancyPercent=35", \
+    "-XX:MetaspaceSize=96m", \
+    "-XX:MinMetaspaceFreeRatio=50", \
+    "-XX:MaxMetaspaceFreeRatio=80", \
+    "-XX:+ExplicitGCInvokesConcurrent", \
+    "-XX:InitialRAMPercentage=5.0", \
+    "-XX:MaxRAMPercentage=50.0", \
 	"-jar", \
 	"/home/jpo-conflictmonitor.jar"]
 
-# ENTRYPOINT ["tail", "-f", "/dev/null"]
+# Entrypoint for testing: enables nonlocal JMX on port 10090
+#ENTRYPOINT ["java", \
+#	"-Dcom.sun.management.jmxremote=true", \
+#	"-Dcom.sun.management.jmxremote.local.only=false", \
+#	"-Dcom.sun.management.jmxremote.authenticate=false", \
+#	"-Dcom.sun.management.jmxremote.ssl=false", \
+#    "-Dcom.sun.management.jmxremote.port=10090", \
+#    "-Dcom.sun.management.jmxremote.rmi.port=10090", \
+#    "-Djava.rmi.server.hostname=localhost", \
+#    "-Dlogback.configurationFile=/home/logback.xml", \
+#    "-XX:+UseG1GC", \
+#    "-XX:MaxGCPauseMillis=20", \
+#    "-XX:InitiatingHeapOccupancyPercent=35", \
+#    "-XX:MetaspaceSize=96m", \
+#    "-XX:MinMetaspaceFreeRatio=50", \
+#    "-XX:MaxMetaspaceFreeRatio=80", \
+#    "-XX:+ExplicitGCInvokesConcurrent", \
+#    "-XX:InitialRAMPercentage=5.0", \
+#    "-XX:MaxRAMPercentage=50.0", \
+#    "-jar", \
+#	"/home/jpo-conflictmonitor.jar"]