Merge pull request #89 from usdot-jpo-ode/master

Sync Master into develop

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "jpo-utils"]
 	path = jpo-utils
 	url = https://github.com/usdot-jpo-ode/jpo-utils
+	branch = master

docker-compose.yml

@@ -61,10 +61,24 @@ services:
       - "5555:5555/udp"
       - "6666:6666/udp"
     environment:
-      DOCKER_HOST_IP: ${DOCKER_HOST_IP:?error}
+      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
+      ODE_KAFKA_BROKERS: ${DOCKER_HOST_IP}:9092
       KAFKA_TYPE: ${KAFKA_TYPE}
       CONFLUENT_KEY: ${CONFLUENT_KEY}
       CONFLUENT_SECRET: ${CONFLUENT_SECRET}
+      DATA_SIGNING_ENABLED_RSU: ${DATA_SIGNING_ENABLED_RSU}
+      DATA_SIGNING_ENABLED_SDW: ${DATA_SIGNING_ENABLED_SDW}
+      DEFAULT_SNMP_PROTOCOL: ${DEFAULT_SNMP_PROTOCOL}
+      KAFKA_LINGER_MS: ${KAFKA_LINGER_MS}
+      KAFKA_ACKS: ${KAFKA_ACKS}
+      KAFKA_RETRIES: ${KAFKA_RETRIES}
+      KAFKA_BATCH_SIZE: ${KAFKA_BATCH_SIZE}
+      KAFKA_COMPRESSION_TYPE: ${KAFKA_COMPRESSION_TYPE}
+      KAFKA_KEY_SERIALIZER: ${KAFKA_KEY_SERIALIZER}
+      KAFKA_VALUE_SERIALIZER: ${KAFKA_VALUE_SERIALIZER}
+      KAFKA_PARTITIONER_CLASS: ${KAFKA_PARTITIONER_CLASS}
+      ODE_TIM_INGEST_MONITORING_ENABLED: ${ODE_TIM_INGEST_MONITORING_ENABLED}
+      ODE_TIM_INGEST_MONITORING_INTERVAL: ${ODE_TIM_INGEST_MONITORING_INTERVAL}
     depends_on:
       kafka:
         condition: service_healthy
@@ -91,7 +105,7 @@ services:
           cpus: '1'
           memory: 2G
     environment:
-      DOCKER_HOST_IP: ${DOCKER_HOST_IP:?error}
+      DOCKER_HOST_IP: ${DOCKER_HOST_IP}
       KAFKA_TYPE: ${KAFKA_TYPE}
       CONFLUENT_KEY: ${CONFLUENT_KEY}
       CONFLUENT_SECRET: ${CONFLUENT_SECRET}

jpo-geojsonconverter/src/main/resources/schemas/map.schema.json

@@ -138,6 +138,12 @@
                         "recordId",
                         "serialNumber"
                     ]
+                },
+                "asn1": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
                 }
             },
             "required": [
@@ -155,7 +161,8 @@
                 "odePacketID",
                 "odeTimStartDateTime",
                 "recordGeneratedAt",
-                "sanitized"
+                "sanitized",
+                "asn1"
             ]
         },
         "OdeMapPayload": {

jpo-geojsonconverter/src/main/resources/schemas/spat.schema.json

@@ -148,6 +148,12 @@
                 },
                 "sanitized": {
                     "type": "boolean"
+                },
+                "asn1": {
+                    "type": [
+                        "string",
+                        "null"
+                    ]
                 }
             },
             "required": [
@@ -161,7 +167,8 @@
                 "odeReceivedAt",
                 "schemaVersion",
                 "recordGeneratedAt",
-                "sanitized"
+                "sanitized",
+                "asn1"
             ],
             "additionalProperties": false
         },

jpo-geojsonconverter/src/test/resources/json/invalid.bsm.json

@@ -23,13 +23,14 @@
             "serialNumber": 0
         },
         "odeReceivedAt": "2024-08-12T12:32:03.811Z",
-        "schemaVersion": 6,
+        "schemaVersion": 8,
         "maxDurationTime": 0,
         "recordGeneratedAt": "",
         "recordGeneratedBy": "OBU",
         "sanitized": false,
         "odePacketID": "",
         "odeTimStartDateTime": "",
+        "asn1": "",
         "originIp": 5
     },
     "payload": {

jpo-geojsonconverter/src/test/resources/json/invalid.map.json

@@ -25,6 +25,7 @@
 		"odePacketID": "",
 		"odeTimStartDateTime": "",
 		"mapSource": "RSU",
+		"asn1": "",
 		"originIp": "172.18.0.1"
 	},
 	"payload": {

jpo-geojsonconverter/src/test/resources/json/invalid.spat.json

@@ -25,6 +25,7 @@
 		"odePacketID": "",
 		"odeTimStartDateTime": "",
 		"spatSource": "V2X",
+		"asn1": "",
 		"originIp": "172.21.0.1",
 		"isCertPresent": false,
 		"invalidExtraProperty": "invalid"

jpo-geojsonconverter/src/test/resources/json/invalid2.map.json

@@ -25,6 +25,7 @@
 		"odePacketID": "",
 		"odeTimStartDateTime": "",
 		"mapSource": "RSU",
+		"asn1": "",
 		"originIp": "172.18.0.1"
 	},
 	"payload": {

jpo-geojsonconverter/src/test/resources/json/valid.bsm.json

@@ -30,6 +30,7 @@
         "sanitized": false,
         "odePacketID": "",
         "odeTimStartDateTime": "",
+        "asn1": "",
         "originIp": "172.19.0.1"
     },
     "payload": {

jpo-geojsonconverter/src/test/resources/json/valid.map.json

@@ -25,6 +25,7 @@
 		"odePacketID": "",
 		"odeTimStartDateTime": "",
 		"mapSource": "RSU",
+		"asn1": "",
 		"originIp": "172.18.0.1"
 	},
 	"payload": {

jpo-geojsonconverter/src/test/resources/json/valid.spat.json

@@ -25,6 +25,7 @@
 		"odePacketID": "",
 		"odeTimStartDateTime": "",
 		"spatSource": "V2X",
+		"asn1": "",
 		"originIp": "172.21.0.1",
 		"isCertPresent": false
 	},

sample.env

@@ -21,6 +21,7 @@ DOCKER_HOST_IP=
 
 # GitHub properties for pulling the latest version of the JPO-ODE at build time
 # NOTE: These variables are only required for building the geojson-converter image, not for running the image
+MAVEN_GITHUB_TOKEN_NAME=
 MAVEN_GITHUB_TOKEN=
 MAVEN_GITHUB_ORG=usdot-jpo-ode
 
@@ -29,12 +30,9 @@ RESTART_POLICY="no"
 
 # Available profiles:
 # - all
-#   - geojson_full
-#       - geojson_converter
-#       - ode
-#       - adm
 #   - geojson_base
 #       - geojson_converter
+#   - geojson_cc_base
 #   - mongo_full
 #       - mongo
 #       - mongo_express
@@ -50,24 +48,151 @@ RESTART_POLICY="no"
 #  - kafka_connect
 #       - kafka_connect
 # EXAMPLE: COMPOSE_PROFILES=kafka_connect_standalone,kafka_ui,mongo_express
-COMPOSE_PROFILES=geojson_full,kafka,kafka_setup
+COMPOSE_PROFILES=geojson_full,kafka,kafka_setup,kafka_ui
 
-####################
-# Confluent Cloud Properties (for Confluent Cloud support)
+# Feature geometry output mode
+# Options are GEOJSON_ONLY or WKT, defaults to GEOJSON_ONLY if nothing is specified
+# WKT mode will still generate geoJSON output
+GEOMETRY_OUTPUT_MODE=
+
+#########################
+# Kafka and Confluent Cloud Properties - START
 
-# Set to "CONFLUENT" if broker is a Confluent Cloud broker
+# The type of Kafka broker to connect to. If set to "CONFLUENT", the broker will be Confluent Cloud. Otherwise, it will be a local Kafka broker.
 KAFKA_TYPE=
+KAFKA_LINGER_MS=1
+KAFKA_BOOTSTRAP_SERVERS=${DOCKER_HOST_IP}:9092
+KAFKA_LOG_RETENTION_HOURS=3
+KAFKA_LOG_RETENTION_BYTES=10737418240 # 10GB
+
+# Local Kafka broker properties - you can override these if you want, but the defaults should be fine for normal use
+# The application will not boot if these environment variables are set to empty strings (e.g. KAFKA_BATCH_SIZE=)
+KAFKA_COMPRESSION_TYPE="zstd"
+KAFKA_BATCH_SIZE=16384
+KAFKA_ACKS="all"
+KAFKA_RETRIES=0
+KAFKA_KEY_SERIALIZER="org.apache.kafka.common.serialization.StringSerializer"
+KAFKA_VALUE_SERIALIZER="org.apache.kafka.common.serialization.StringSerializer"
+KAFKA_PARTITIONER_CLASS="org.apache.kafka.clients.producer.internals.DefaultPartitioner"
 
-# Set to actual Confluent Cloud access key and secret values for SASL authentication
+# Variables for creating kafka topics:
+KAFKA_TOPIC_PARTITIONS=1
+KAFKA_TOPIC_REPLICAS=1
+KAFKA_TOPIC_MIN_INSYNC_REPLICAS=1
+KAFKA_TOPIC_RETENTION_MS=300000
+KAFKA_TOPIC_DELETE_RETENTION_MS=3600000
+KAFKA_TOPIC_CREATE_ODE=true                 # Create topics for ODE
+KAFKA_TOPIC_CREATE_GEOJSONCONVERTER=true    # Create topics for GeoJSON Converter
+KAFKA_TOPIC_CREATE_CONFLICTMONITOR=true     # Create topics for Conflict Monitor
+KAFKA_TOPIC_CREATE_DEDUPLICATOR=false       # Create topics for Deduplicator
+KAFKA_TOPIC_CREATE_MECDEPOSIT=false         # Create topics for MecDeposit
+# Relative path to the Kafka topic yaml configuration script, upper level directories are supported
+# NOTE: This script is used to create kafka topics
+KAFKA_TOPIC_CONFIG_RELATIVE_PATH="./jikkou/kafka-topics-values.yaml"
+
+# Set to Confluent Cloud access key and secret values for SASL authentication
 CONFLUENT_KEY=
 CONFLUENT_SECRET=
 
-# Feature geometry output mode
-# Options are GEOJSON_ONLY or WKT, defaults to GEOJSON_ONLY if nothing is specified
-# WKT mode will still generate geoJSON output
-GEOMETRY_OUTPUT_MODE=
+# Kafka and Confluent Cloud Properties - END
+#########################
+
+#########################
+# JPO-ODE Specific Properties - START
+
+# These are optional
+DATA_SIGNING_ENABLED_RSU=
+DATA_SIGNING_ENABLED_SDW=
+DEFAULT_SNMP_PROTOCOL=
+
+# ODE Monitoring
+ODE_TIM_INGEST_MONITORING_ENABLED=false
+# The interval is measured in seconds. 60 seconds is a sane default for local monitoring, but you may want to increase
+# the interval to 3600 (1 hour) or more for production monitoring to reduce the noise in logs for healthy systems
+ODE_TIM_INGEST_MONITORING_INTERVAL=60
+
+# JPO-ODE Specific Properties - END
+#########################
+
+#########################
+# ACM Logging Properties - START
 
-# ACM Logging
 ADM_LOG_TO_FILE=false
 ADM_LOG_TO_CONSOLE=true
-ADM_LOG_LEVEL=INFO
+ADM_LOG_LEVEL=INFO
+
+# ACM Logging Properties - END
+#########################
+
+#########################
+# MongoDB Properties - START
+
+# NOTE: Must set a password for the container to start up properly
+MONGO_IP=${DOCKER_HOST_IP}
+MONGO_DB_NAME=CV
+
+# Generate a random string for the MongoDB keyfile using the following command:
+# $ openssl rand -base64 32
+MONGO_DB_KEYFILE_STRING=replacethisstring
+
+MONGO_ADMIN_DB_USER=admin
+MONGO_ADMIN_DB_PASS=replace_me
+
+MONGO_READ_WRITE_USER=ode
+MONGO_READ_WRITE_PASS=replace_me
+
+MONGO_READ_USER=user
+MONGO_READ_PASS=replace_me
+
+MONGO_EXPORTER_USERNAME=export
+MONGO_EXPORTER_PASSWORD=replace_me
+
+MONGO_EXPRESS_USER=${MONGO_ADMIN_DB_USER}
+MONGO_EXPRESS_PASS=${MONGO_ADMIN_DB_PASS}
+
+MONGO_PORT=27017
+MONGO_DATA_RETENTION_SECONDS=5184000
+MONGO_ASN_RETENTION_SECONDS=86400
+
+
+MONGO_DATABASE_STORAGE_COLLECTION_NAME=MongoStorage
+MONGO_DATABASE_SIZE_GB=1000
+MONGO_DATABASE_SIZE_TARGET_PERCENT=0.8
+MONGO_DATABASE_DELETE_THRESHOLD_PERCENT=0.9
+MONGO_DATABASE_MAX_TTL_RETENTION_SECONDS=5184000
+MONGO_DATABASE_MIN_TTL_RETENTION_SECONDS=604800
+MONGO_DATABASE_COMPACTION_TRIGGER_PERCENT=MONGO_DATABASE_COMPACTION_TRIGGER_PERCENT
+MONGO_ENABLE_STORAGE_RECORD=true
+MONGO_ENABLE_DYNAMIC_TTL=true
+
+
+
+# Relative path to the MongoDB init script, upper level directories are supported
+MONGO_SETUP_SCRIPT_RELATIVE_PATH="./mongo/setup_mongo.sh"
+MONGO_INIT_REPLICAS_SCRIPT_RELATIVE_PATH="./mongo/init_replicas.js"
+MONGO_CREATE_INDEXES_SCRIPT_RELATIVE_PATH="./mongo/create_indexes.js"
+MONGO_MANAGE_VOLUMES_SCRIPT_RELATIVE_PATH="./mongo/manage_volume.js"
+
+# MongoDB Properties - END
+#########################
+
+#########################
+# Kafka Connect Properties - START
+
+# NOTE: Required variables: [MONGODB, KAFKA]
+CONNECT_URL=http://${DOCKER_HOST_IP}:8083
+# Kafka connect log level
+CONNECT_LOG_LEVEL=ERROR
+
+CONNECT_TASKS_MAX=1                     # Number of concurrent tasks to configure on kafka connectors
+CONNECT_CREATE_ODE=true                 # Create kafka connectors to MongoDB for ODE
+CONNECT_CREATE_GEOJSONCONVERTER=true    # Create kafka connectors to MongoDB for GeoJSON Converter
+CONNECT_CREATE_CONFLICTMONITOR=true     # Create kafka connectors to MongoDB for Conflict Monitor
+CONNECT_CREATE_DEDUPLICATOR=false       # Create kafka connectors to MongoDB for Deduplicator
+CONNECT_CREATE_MECDEPOSIT=false         # Create kafka connectors to MongoDB for MecDeposit
+# Relative path to the Kafka Connector yaml configuration script, upper level directories are supported
+# NOTE: This script is used to create kafka connectors
+CONNECT_CONFIG_RELATIVE_PATH="./jikkou/kafka-connectors-values.yaml"
+
+# Kafka Connect - END
+#########################