From 39977319dc2c9e9de95354007d6a9e04a5a296db Mon Sep 17 00:00:00 2001 From: nscuro Date: Tue, 26 Dec 2023 14:51:11 +0100 Subject: [PATCH 01/26] Move Kafka Streams classes to dedicated `streams` package Signed-off-by: nscuro --- .../KafkaStreamsInitializer.java | 8 +++---- .../KafkaStreamsTopologyFactory.java | 23 +++++++------------ ...bstractThresholdBasedExceptionHandler.java | 2 +- ...treamsDeserializationExceptionHandler.java | 2 +- ...afkaStreamsProductionExceptionHandler.java | 2 +- .../KafkaStreamsUncaughtExceptionHandler.java | 2 +- ...ayedBomProcessedNotificationProcessor.java | 2 +- .../MirrorVulnerabilityProcessor.java | 2 +- .../RepositoryMetaResultProcessor.java | 2 +- .../VulnerabilityScanResultProcessor.java | 2 +- .../health/KafkaStreamsHealthCheck.java | 2 +- src/main/webapp/WEB-INF/web.xml | 2 +- ...msDelayedBomProcessedNotificationTest.java | 3 ++- .../KafkaStreamsPostgresTest.java | 3 ++- .../kafka/{ => streams}/KafkaStreamsTest.java | 3 ++- .../KafkaStreamsTopologyTest.java | 3 ++- ...msDeserializationExceptionHandlerTest.java | 2 +- ...StreamsProductionExceptionHandlerTest.java | 2 +- ...kaStreamsUncaughtExceptionHandlerTest.java | 2 +- .../MirrorVulnerabilityProcessorTest.java | 2 +- .../RepositoryMetaResultProcessorTest.java | 2 +- .../VulnerabilityScanResultProcessorTest.java | 2 +- 22 files changed, 36 insertions(+), 39 deletions(-) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/KafkaStreamsInitializer.java (95%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/KafkaStreamsTopologyFactory.java (93%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/exception/AbstractThresholdBasedExceptionHandler.java (96%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/exception/KafkaStreamsDeserializationExceptionHandler.java (98%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/exception/KafkaStreamsProductionExceptionHandler.java (98%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/exception/KafkaStreamsUncaughtExceptionHandler.java (98%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/processor/DelayedBomProcessedNotificationProcessor.java (98%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/processor/MirrorVulnerabilityProcessor.java (99%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/processor/RepositoryMetaResultProcessor.java (99%) rename src/main/java/org/dependencytrack/event/kafka/{ => streams}/processor/VulnerabilityScanResultProcessor.java (99%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/KafkaStreamsDelayedBomProcessedNotificationTest.java (99%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/KafkaStreamsPostgresTest.java (97%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/KafkaStreamsTest.java (96%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/KafkaStreamsTopologyTest.java (99%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/exception/KafkaStreamsDeserializationExceptionHandlerTest.java (97%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/exception/KafkaStreamsProductionExceptionHandlerTest.java (97%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/exception/KafkaStreamsUncaughtExceptionHandlerTest.java (97%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/processor/MirrorVulnerabilityProcessorTest.java (99%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/processor/RepositoryMetaResultProcessorTest.java (99%) rename src/test/java/org/dependencytrack/event/kafka/{ => streams}/processor/VulnerabilityScanResultProcessorTest.java (99%) diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaStreamsInitializer.java b/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsInitializer.java similarity index 95% rename from src/main/java/org/dependencytrack/event/kafka/KafkaStreamsInitializer.java rename to src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsInitializer.java index 48d70695d..4358822d6 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaStreamsInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsInitializer.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka; +package org.dependencytrack.event.kafka.streams; import alpine.Config; import alpine.common.logging.Logger; @@ -11,9 +11,9 @@ import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsConfig; import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.kafka.exception.KafkaStreamsDeserializationExceptionHandler; -import org.dependencytrack.event.kafka.exception.KafkaStreamsProductionExceptionHandler; -import org.dependencytrack.event.kafka.exception.KafkaStreamsUncaughtExceptionHandler; +import org.dependencytrack.event.kafka.streams.exception.KafkaStreamsDeserializationExceptionHandler; +import org.dependencytrack.event.kafka.streams.exception.KafkaStreamsProductionExceptionHandler; +import org.dependencytrack.event.kafka.streams.exception.KafkaStreamsUncaughtExceptionHandler; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaStreamsTopologyFactory.java b/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java similarity index 93% rename from src/main/java/org/dependencytrack/event/kafka/KafkaStreamsTopologyFactory.java rename to src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java index 2b6cb2e38..12a5b6b15 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaStreamsTopologyFactory.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka; +package org.dependencytrack.event.kafka.streams; import alpine.Config; import alpine.common.logging.Logger; @@ -9,22 +9,15 @@ import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; -import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Named; -import org.apache.kafka.streams.kstream.Produced; -import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.kstream.*; import org.datanucleus.PropertyNames; import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.ComponentMetricsUpdateEvent; -import org.dependencytrack.event.ComponentPolicyEvaluationEvent; -import org.dependencytrack.event.PortfolioVulnerabilityAnalysisEvent; -import org.dependencytrack.event.ProjectMetricsUpdateEvent; -import org.dependencytrack.event.ProjectPolicyEvaluationEvent; -import org.dependencytrack.event.kafka.processor.DelayedBomProcessedNotificationProcessor; -import org.dependencytrack.event.kafka.processor.MirrorVulnerabilityProcessor; -import org.dependencytrack.event.kafka.processor.RepositoryMetaResultProcessor; -import org.dependencytrack.event.kafka.processor.VulnerabilityScanResultProcessor; +import org.dependencytrack.event.*; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.event.kafka.streams.processor.DelayedBomProcessedNotificationProcessor; +import org.dependencytrack.event.kafka.streams.processor.MirrorVulnerabilityProcessor; +import org.dependencytrack.event.kafka.streams.processor.RepositoryMetaResultProcessor; +import org.dependencytrack.event.kafka.streams.processor.VulnerabilityScanResultProcessor; import org.dependencytrack.model.VulnerabilityScan; import org.dependencytrack.model.WorkflowState; import org.dependencytrack.model.WorkflowStatus; diff --git a/src/main/java/org/dependencytrack/event/kafka/exception/AbstractThresholdBasedExceptionHandler.java b/src/main/java/org/dependencytrack/event/kafka/streams/exception/AbstractThresholdBasedExceptionHandler.java similarity index 96% rename from src/main/java/org/dependencytrack/event/kafka/exception/AbstractThresholdBasedExceptionHandler.java rename to src/main/java/org/dependencytrack/event/kafka/streams/exception/AbstractThresholdBasedExceptionHandler.java index 5089560ef..b11705602 100644 --- a/src/main/java/org/dependencytrack/event/kafka/exception/AbstractThresholdBasedExceptionHandler.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/exception/AbstractThresholdBasedExceptionHandler.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import java.time.Clock; import java.time.Duration; diff --git a/src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsDeserializationExceptionHandler.java b/src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsDeserializationExceptionHandler.java similarity index 98% rename from src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsDeserializationExceptionHandler.java rename to src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsDeserializationExceptionHandler.java index 83bc9dd50..e91c9a38c 100644 --- a/src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsDeserializationExceptionHandler.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsDeserializationExceptionHandler.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import alpine.Config; import alpine.common.logging.Logger; diff --git a/src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsProductionExceptionHandler.java b/src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsProductionExceptionHandler.java similarity index 98% rename from src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsProductionExceptionHandler.java rename to src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsProductionExceptionHandler.java index afae44abd..37cc45963 100644 --- a/src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsProductionExceptionHandler.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsProductionExceptionHandler.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import alpine.Config; import alpine.common.logging.Logger; diff --git a/src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsUncaughtExceptionHandler.java b/src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsUncaughtExceptionHandler.java similarity index 98% rename from src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsUncaughtExceptionHandler.java rename to src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsUncaughtExceptionHandler.java index c862ddd3b..492827890 100644 --- a/src/main/java/org/dependencytrack/event/kafka/exception/KafkaStreamsUncaughtExceptionHandler.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsUncaughtExceptionHandler.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import alpine.Config; import alpine.common.logging.Logger; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/streams/processor/DelayedBomProcessedNotificationProcessor.java similarity index 98% rename from src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/streams/processor/DelayedBomProcessedNotificationProcessor.java index cddc5b692..5df73b646 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/processor/DelayedBomProcessedNotificationProcessor.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import alpine.common.logging.Logger; import alpine.notification.NotificationLevel; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/MirrorVulnerabilityProcessor.java b/src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java similarity index 99% rename from src/main/java/org/dependencytrack/event/kafka/processor/MirrorVulnerabilityProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java index 123f8e739..ed80d4d4d 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/MirrorVulnerabilityProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import alpine.common.logging.Logger; import alpine.common.metrics.Metrics; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessor.java similarity index 99% rename from src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessor.java index 8a4eb6f5a..7b2bace7a 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessor.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import alpine.common.logging.Logger; import alpine.common.metrics.Metrics; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessor.java similarity index 99% rename from src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessor.java index 7e16f8385..64e051762 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessor.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import alpine.Config; import alpine.common.logging.Logger; diff --git a/src/main/java/org/dependencytrack/health/KafkaStreamsHealthCheck.java b/src/main/java/org/dependencytrack/health/KafkaStreamsHealthCheck.java index c3553b3dd..730d47337 100644 --- a/src/main/java/org/dependencytrack/health/KafkaStreamsHealthCheck.java +++ b/src/main/java/org/dependencytrack/health/KafkaStreamsHealthCheck.java @@ -1,7 +1,7 @@ package org.dependencytrack.health; import org.apache.kafka.streams.KafkaStreams; -import org.dependencytrack.event.kafka.KafkaStreamsInitializer; +import org.dependencytrack.event.kafka.streams.KafkaStreamsInitializer; import org.eclipse.microprofile.health.HealthCheck; import org.eclipse.microprofile.health.HealthCheckResponse; import org.eclipse.microprofile.health.Liveness; diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 74af896da..f7f1c5d63 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -48,7 +48,7 @@ org.dependencytrack.event.EventSubsystemInitializer - org.dependencytrack.event.kafka.KafkaStreamsInitializer + org.dependencytrack.event.kafka.streams.KafkaStreamsInitializer org.dependencytrack.event.PurlMigrator diff --git a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsDelayedBomProcessedNotificationTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsDelayedBomProcessedNotificationTest.java similarity index 99% rename from src/test/java/org/dependencytrack/event/kafka/KafkaStreamsDelayedBomProcessedNotificationTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsDelayedBomProcessedNotificationTest.java index 758e4b47d..c84db7b3c 100644 --- a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsDelayedBomProcessedNotificationTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsDelayedBomProcessedNotificationTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka; +package org.dependencytrack.event.kafka.streams; import net.mguenther.kafka.junit.KeyValue; import net.mguenther.kafka.junit.ReadKeyValues; @@ -6,6 +6,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.StringDeserializer; +import org.dependencytrack.event.kafka.KafkaTopics; import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; import org.dependencytrack.model.Project; import org.dependencytrack.model.VulnerabilityScan; diff --git a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsPostgresTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsPostgresTest.java similarity index 97% rename from src/test/java/org/dependencytrack/event/kafka/KafkaStreamsPostgresTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsPostgresTest.java index 1122a3325..b0dc20657 100644 --- a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsPostgresTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsPostgresTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka; +package org.dependencytrack.event.kafka.streams; import net.mguenther.kafka.junit.ExternalKafkaCluster; import net.mguenther.kafka.junit.TopicConfig; @@ -6,6 +6,7 @@ import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; import org.dependencytrack.AbstractPostgresEnabledTest; +import org.dependencytrack.event.kafka.KafkaTopics; import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; import org.dependencytrack.proto.notification.v1.Notification; import org.junit.After; diff --git a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java similarity index 96% rename from src/test/java/org/dependencytrack/event/kafka/KafkaStreamsTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java index 6de799594..1d3fb7cf9 100644 --- a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java @@ -1,10 +1,11 @@ -package org.dependencytrack.event.kafka; +package org.dependencytrack.event.kafka.streams; import net.mguenther.kafka.junit.ExternalKafkaCluster; import net.mguenther.kafka.junit.TopicConfig; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsConfig; import org.dependencytrack.PersistenceCapableTest; +import org.dependencytrack.event.kafka.KafkaTopics; import org.junit.After; import org.junit.Before; import org.junit.Rule; diff --git a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsTopologyTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyTest.java similarity index 99% rename from src/test/java/org/dependencytrack/event/kafka/KafkaStreamsTopologyTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyTest.java index 79bccb001..e3c632f93 100644 --- a/src/test/java/org/dependencytrack/event/kafka/KafkaStreamsTopologyTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka; +package org.dependencytrack.event.kafka.streams; import alpine.event.framework.Event; import alpine.event.framework.EventService; @@ -19,6 +19,7 @@ import org.dependencytrack.event.PortfolioVulnerabilityAnalysisEvent; import org.dependencytrack.event.ProjectMetricsUpdateEvent; import org.dependencytrack.event.ProjectPolicyEvaluationEvent; +import org.dependencytrack.event.kafka.KafkaTopics; import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; import org.dependencytrack.model.Policy; import org.dependencytrack.model.PolicyCondition; diff --git a/src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsDeserializationExceptionHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsDeserializationExceptionHandlerTest.java similarity index 97% rename from src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsDeserializationExceptionHandlerTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsDeserializationExceptionHandlerTest.java index 4ce0df779..43aac8de9 100644 --- a/src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsDeserializationExceptionHandlerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsDeserializationExceptionHandlerTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; diff --git a/src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsProductionExceptionHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsProductionExceptionHandlerTest.java similarity index 97% rename from src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsProductionExceptionHandlerTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsProductionExceptionHandlerTest.java index 0e2c8e6b4..741933f92 100644 --- a/src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsProductionExceptionHandlerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsProductionExceptionHandlerTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.errors.RecordTooLargeException; diff --git a/src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsUncaughtExceptionHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsUncaughtExceptionHandlerTest.java similarity index 97% rename from src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsUncaughtExceptionHandlerTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsUncaughtExceptionHandlerTest.java index 992330edf..4ee06fe3e 100644 --- a/src/test/java/org/dependencytrack/event/kafka/exception/KafkaStreamsUncaughtExceptionHandlerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/exception/KafkaStreamsUncaughtExceptionHandlerTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.exception; +package org.dependencytrack.event.kafka.streams.exception; import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse; import org.junit.Test; diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/MirrorVulnerabilityProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessorTest.java similarity index 99% rename from src/test/java/org/dependencytrack/event/kafka/processor/MirrorVulnerabilityProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessorTest.java index fb0ae342a..b8a57778c 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/MirrorVulnerabilityProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessorTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.StringSerializer; diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessorTest.java similarity index 99% rename from src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessorTest.java index 16506299a..5f14df992 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessorTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import com.google.protobuf.Timestamp; import org.apache.kafka.common.serialization.StringDeserializer; diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessorTest.java similarity index 99% rename from src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessorTest.java index 4945ca0b4..364407e13 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessorTest.java @@ -1,4 +1,4 @@ -package org.dependencytrack.event.kafka.processor; +package org.dependencytrack.event.kafka.streams.processor; import com.google.protobuf.Timestamp; import junitparams.JUnitParamsRunner; From 021d87b9238d120cf12b55f27102803766efb745 Mon Sep 17 00:00:00 2001 From: nscuro Date: Wed, 27 Dec 2023 16:45:06 +0100 Subject: [PATCH 02/26] WIP: Add mechanism to instantiate and manage parallel consumers Signed-off-by: nscuro --- pom.xml | 8 + .../kafka/processor/KafkaProcessorConfig.java | 85 ++++++ .../processor/KafkaProcessorHealthCheck.java | 45 +++ .../processor/KafkaProcessorInitializer.java | 33 ++ .../processor/KafkaProcessorManager.java | 284 ++++++++++++++++++ .../kafka/processor/KafkaRecordHandler.java | 24 ++ .../health/HealthCheckInitializer.java | 2 + .../processor/KafkaProcessorManagerTest.java | 27 ++ 8 files changed, 508 insertions(+) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java create mode 100644 src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java diff --git a/pom.xml b/pom.xml index a8887d24d..1f799e230 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ 7.3 0.2.2 1.4.1 + 0.5.2.7 3.2.0 3.25.1 1.18.3 @@ -300,6 +301,13 @@ packageurl-java ${lib.packageurl.version} + + + io.confluent.parallelconsumer + parallel-consumer-core + ${lib.parallel-consumer.version} + + org.apache.kafka kafka-clients diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java new file mode 100644 index 000000000..033f57e07 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java @@ -0,0 +1,85 @@ +package org.dependencytrack.event.kafka.processor; + +import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; + +import java.lang.reflect.Method; +import java.util.Collection; + +public class KafkaProcessorConfig { + + private String name; + private Collection topics; + private Object handlerInstance; + private Method handlerMethod; + private ProcessingOrder ordering; + private int maxBatchSize; + private int maxConcurrency; + + public String name() { + return name; + } + + public KafkaProcessorConfig setName(String name) { + this.name = name; + return this; + } + + public Collection topics() { + return topics; + } + + public KafkaProcessorConfig setTopics(Collection topics) { + this.topics = topics; + return this; + } + + public Object handlerInstance() { + return handlerInstance; + } + + public KafkaProcessorConfig setHandlerInstance(Object handlerInstance) { + this.handlerInstance = handlerInstance; + return this; + } + + public Method handlerMethod() { + return handlerMethod; + } + + public KafkaProcessorConfig setHandlerMethod(Method handlerMethod) { + this.handlerMethod = handlerMethod; + return this; + } + + public ProcessingOrder ordering() { + return ordering; + } + + public KafkaProcessorConfig setOrdering(ProcessingOrder ordering) { + this.ordering = ordering; + return this; + } + + public int maxBatchSize() { + return maxBatchSize; + } + + public KafkaProcessorConfig setMaxBatchSize(int maxBatchSize) { + this.maxBatchSize = maxBatchSize; + return this; + } + + public int maxConcurrency() { + return maxConcurrency; + } + + public KafkaProcessorConfig setMaxConcurrency(int maxConcurrency) { + this.maxConcurrency = maxConcurrency; + return this; + } + + public boolean isBatch() { + return maxBatchSize > 1; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java new file mode 100644 index 000000000..b676b8e97 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java @@ -0,0 +1,45 @@ +package org.dependencytrack.event.kafka.processor; + +import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import org.eclipse.microprofile.health.HealthCheck; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.eclipse.microprofile.health.HealthCheckResponseBuilder; +import org.eclipse.microprofile.health.Liveness; + +import java.util.Map; + +@Liveness +public class KafkaProcessorHealthCheck implements HealthCheck { + + @Override + public HealthCheckResponse call() { + final KafkaProcessorManager processorManager = KafkaProcessorInitializer.processorManager(); + if (processorManager == null) { + return HealthCheckResponse.builder().down().build(); + } + + boolean isUp = true; + final HealthCheckResponseBuilder responseBuilder = HealthCheckResponse.named("kafka-processor"); + for (final Map.Entry> processorByName : processorManager.processors().entrySet()) { + final String processorName = processorByName.getKey(); + final ParallelStreamProcessor processor = processorByName.getValue(); + + final boolean isProcessorUp = !processor.isClosedOrFailed(); + responseBuilder.withData(processorName, isProcessorUp + ? HealthCheckResponse.Status.UP.name() + : HealthCheckResponse.Status.DOWN.name()); + + if (!isProcessorUp + && processor instanceof final ParallelEoSStreamProcessor concreteProcessor + && concreteProcessor.getFailureCause() != null) { + isUp = false; + responseBuilder.withData("%s_failure_cause".formatted(processorName), + concreteProcessor.getFailureCause().getMessage()); + } + } + + return responseBuilder.status(isUp).build(); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java new file mode 100644 index 000000000..09f6adc3b --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java @@ -0,0 +1,33 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.Config; +import alpine.common.logging.Logger; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +public class KafkaProcessorInitializer implements ServletContextListener { + + private static final Logger LOGGER = Logger.getLogger(KafkaProcessorInitializer.class); + private static KafkaProcessorManager PROCESSOR_MANAGER; + + @Override + public void contextInitialized(ServletContextEvent sce) { + PROCESSOR_MANAGER = new KafkaProcessorManager(Config.getInstance()); + LOGGER.info("Initializing Kafka processors (instance ID: %s)".formatted(PROCESSOR_MANAGER.getInstanceId())); + } + + @Override + public void contextDestroyed(ServletContextEvent sce) { + LOGGER.info("Stopping Kafka processors"); + if (PROCESSOR_MANAGER != null) { + PROCESSOR_MANAGER.close(); + PROCESSOR_MANAGER = null; + } + } + + static KafkaProcessorManager processorManager() { + return PROCESSOR_MANAGER; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java new file mode 100644 index 000000000..9c134ee97 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java @@ -0,0 +1,284 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.Config; +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import io.confluent.parallelconsumer.ParallelConsumerOptions; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.serialization.VoidDeserializer; +import org.apache.kafka.common.serialization.VoidSerializer; +import org.cyclonedx.proto.v1_4.Bom; +import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; +import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; +import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; +import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; + +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static alpine.Config.AlpineKey.METRICS_ENABLED; +import static io.confluent.parallelconsumer.ParallelStreamProcessor.createEosStreamProcessor; +import static java.util.Map.entry; +import static java.util.Objects.requireNonNull; +import static org.apache.commons.lang3.reflect.MethodUtils.getMethodsListWithAnnotation; +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.CLIENT_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; +import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; + +public class KafkaProcessorManager implements AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(KafkaProcessorManager.class); + private static final Map> SERIALIZERS = Map.ofEntries( + entry("void", new VoidSerializer()), + entry(String.class.getTypeName(), new StringSerializer()), + entry(ScanKey.class.getTypeName(), new KafkaProtobufSerializer<>()), + entry(ScanResult.class.getTypeName(), new KafkaProtobufSerializer<>()) + ); + private static final Map> DESERIALIZERS = Map.ofEntries( + entry("void", new VoidDeserializer()), + entry(String.class.getTypeName(), new StringDeserializer()), + entry(Bom.class.getTypeName(), new KafkaProtobufDeserializer<>(Bom.parser())), + entry(ScanKey.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanKey.parser())), + entry(ScanResult.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanResult.parser())) + ); + + private final UUID instanceId = UUID.randomUUID(); + private final Map> processors = new HashMap<>(); + + private final Config config; + + KafkaProcessorManager(final Config config) { + this.config = config; + } + + public void registerHandler(final Object handler) { + requireNonNull(handler); + + final List processorConfigs = + getMethodsListWithAnnotation(handler.getClass(), KafkaRecordHandler.class).stream() + .map(method -> processHandlerMethod(handler, method)) + .toList(); + if (processorConfigs.isEmpty()) { + throw new IllegalArgumentException(""); + } + + for (final KafkaProcessorConfig processorConfig : processorConfigs) { + final ParallelStreamProcessor processor = + processors.computeIfAbsent(processorConfig.name(), ignored -> createProcessor(processorConfig)); + processor.poll(pollCtx -> { + + }); + } + } + + public UUID getInstanceId() { + return instanceId; + } + + public Map> processors() { + return Collections.unmodifiableMap(processors); + } + + @Override + public void close() { + processors.forEach((processorName, processor) -> { + LOGGER.info("Closing processor %s".formatted(processorName)); + processor.closeDontDrainFirst(); + }); + } + + private KafkaProcessorConfig processHandlerMethod(final Object instance, final Method method) { + final Type[] parameterTypes = method.getGenericParameterTypes(); + if (parameterTypes.length != 1) { + throw new IllegalArgumentException("Handler methods must accepts exactly one parameter, but %s is expecting %d" + .formatted(method.getName(), parameterTypes.length)); + } + + final Type parameterType = parameterTypes[0]; + if (!isConsumerRecord(parameterType) && !isConsumerRecordList(parameterType)) { + throw new IllegalArgumentException(""" + Handler methods must accept one or multiple %ss, but %s is expecting %s instead\ + """.formatted(ConsumerRecord.class.getTypeName(), method.getName(), parameterType.getTypeName())); + } + + final Type returnType = method.getGenericReturnType(); + if (!isVoid(returnType) && !isProducerRecord(returnType) && !isProducerRecordList(returnType)) { + throw new IllegalArgumentException(""" + Handler methods must return either %s, or one or multiple %ss, but %s is returning %s instead\ + """.formatted(Void.TYPE.getTypeName(), ProducerRecord.class.getTypeName(), method.getName(), returnType.getTypeName())); + } + + final Map.Entry consumerKeyValueTypes; + if (isConsumerRecord(parameterType)) { + final var paramType = (ParameterizedType) parameterType; + consumerKeyValueTypes = Map.entry(paramType.getActualTypeArguments()[0], paramType.getActualTypeArguments()[1]); + } else if (isConsumerRecordList(parameterType)) { + final var listParamType = (ParameterizedType) parameterType; + final var recordParamType = (ParameterizedType) listParamType.getActualTypeArguments()[0]; + consumerKeyValueTypes = Map.entry(recordParamType.getActualTypeArguments()[0], recordParamType.getActualTypeArguments()[1]); + } else { + throw new IllegalStateException(""); + } + + if (!DESERIALIZERS.containsKey(consumerKeyValueTypes.getKey().getTypeName())) { + throw new IllegalStateException("No deserializer known for key type %s" + .formatted(consumerKeyValueTypes.getKey().getTypeName())); + } + if (!DESERIALIZERS.containsKey(consumerKeyValueTypes.getValue().getTypeName())) { + throw new IllegalStateException("No deserializer known for value type %s" + .formatted(consumerKeyValueTypes.getValue().getTypeName())); + } + + final KafkaRecordHandler annotation = method.getAnnotation(KafkaRecordHandler.class); + return new KafkaProcessorConfig() + .setName(annotation.name()) + .setTopics(Arrays.asList(annotation.topics())) + .setHandlerInstance(instance) + .setHandlerMethod(method) + .setMaxBatchSize(annotation.maxBatchSize()) + .setMaxConcurrency(annotation.maxConcurrency()); + } + + private ParallelStreamProcessor createProcessor(final KafkaProcessorConfig processorConfig) { + final var optionsBuilder = ParallelConsumerOptions.builder() + .consumer(createConsumer(processorConfig.name())) + .maxConcurrency(processorConfig.maxConcurrency()) + .ordering(processorConfig.ordering()); + + if (processorConfig.isBatch()) { + optionsBuilder.batchSize(processorConfig.maxBatchSize()); + } + + if (config.getPropertyAsBoolean(METRICS_ENABLED)) { + optionsBuilder + .meterRegistry(Metrics.getRegistry()) + .pcInstanceTag(processorConfig.name()); + } + + final ParallelStreamProcessor processor = createEosStreamProcessor(optionsBuilder.build()); + processor.subscribe(processorConfig.topics()); + + return processor; + } + + private Consumer createConsumer(final String name) { + final var consumerConfig = new HashMap(); + consumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); + consumerConfig.put(CLIENT_ID_CONFIG, "%s-consumer-%s".formatted(name, instanceId)); + consumerConfig.put(GROUP_ID_CONFIG, name); + consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(AUTO_OFFSET_RESET_CONFIG, "latest"); + consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); + + final var propertyPrefix = "kafka.processor.%s.consumer".formatted(name); + final Map extraProperties = config.getPassThroughProperties(propertyPrefix); + for (final Map.Entry property : extraProperties.entrySet()) { + final String propertyName = property.getKey().replaceFirst(propertyPrefix, ""); + if (!ConsumerConfig.configNames().contains(propertyName)) { + LOGGER.warn("Provided property %s is not a known consumer config; Skipping".formatted(propertyName)); + continue; + } + + consumerConfig.put(propertyName, property.getValue()); + } + + final var consumer = new KafkaConsumer(consumerConfig); + if (config.getPropertyAsBoolean(METRICS_ENABLED)) { + new KafkaClientMetrics(consumer).bindTo(Metrics.getRegistry()); + } + + return consumer; + } + + private Producer createProducer(final String name) { + final var producerConfig = new HashMap(); + producerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); + producerConfig.put(CLIENT_ID_CONFIG, "%s-producer-%s".formatted(name, instanceId)); + producerConfig.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + producerConfig.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + + final var propertyPrefix = "kafka.processor.%s.producer".formatted(name); + final Map extraProperties = config.getPassThroughProperties(propertyPrefix); + for (final Map.Entry property : extraProperties.entrySet()) { + final String propertyName = property.getKey().replaceFirst(propertyPrefix, ""); + if (!ProducerConfig.configNames().contains(propertyName)) { + LOGGER.warn("Provided property %s is not a known producer config; Skipping".formatted(propertyName)); + continue; + } + + producerConfig.put(propertyName, property.getValue()); + } + + final var producer = new KafkaProducer(producerConfig); + if (config.getPropertyAsBoolean(METRICS_ENABLED)) { + new KafkaClientMetrics(producer).bindTo(Metrics.getRegistry()); + } + + return producer; + } + + private static boolean isConsumerRecord(final Type type) { + // ConsumerRecord + return type instanceof final ParameterizedType paramType + && ConsumerRecord.class.getTypeName().equals(paramType.getRawType().getTypeName()) + && paramType.getActualTypeArguments().length == 2; + } + + private static boolean isConsumerRecordList(final Type type) { + // List> + return type instanceof final ParameterizedType paramType + && List.class.getTypeName().equals(paramType.getRawType().getTypeName()) + && paramType.getActualTypeArguments().length == 1 + && isConsumerRecord(paramType.getActualTypeArguments()[0]); + } + + private static boolean isProducerRecord(final Type type) { + // ProducerRecord + return type instanceof final ParameterizedType paramType + && ProducerRecord.class.getTypeName().equals(paramType.getRawType().getTypeName()) + && paramType.getActualTypeArguments().length == 2; + } + + private static boolean isProducerRecordList(final Type type) { + // List> + return type instanceof final ParameterizedType paramType + && List.class.getTypeName().equals(paramType.getRawType().getTypeName()) + && paramType.getActualTypeArguments().length == 1 + && isProducerRecord(paramType.getActualTypeArguments()[0]); + } + + private static boolean isVoid(final Type type) { + return Void.TYPE.getTypeName().equals(type.getTypeName()); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java new file mode 100644 index 000000000..a00a77ac5 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java @@ -0,0 +1,24 @@ +package org.dependencytrack.event.kafka.processor; + +import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface KafkaRecordHandler { + + String name(); + + String[] topics(); + + ProcessingOrder ordering() default ProcessingOrder.KEY; + + int maxBatchSize() default 1; + + int maxConcurrency() default 1; + +} diff --git a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java index 60ad5dd04..1f6cbc232 100644 --- a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java +++ b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java @@ -24,6 +24,7 @@ import alpine.server.health.checks.DatabaseHealthCheck; import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; +import org.dependencytrack.event.kafka.processor.KafkaProcessorHealthCheck; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -37,6 +38,7 @@ public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Registering health checks"); HealthCheckRegistry.getInstance().register("database", new DatabaseHealthCheck()); HealthCheckRegistry.getInstance().register("kafka-streams", new KafkaStreamsHealthCheck()); + HealthCheckRegistry.getInstance().register("kafka-processor", new KafkaProcessorHealthCheck()); // TODO: Move this to its own initializer if it turns out to be useful LOGGER.info("Registering extra process metrics"); diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java new file mode 100644 index 000000000..47e7a4062 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java @@ -0,0 +1,27 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.Config; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.junit.Test; + +import java.util.List; + +public class KafkaProcessorManagerTest { + + public static class TestHandler { + + @KafkaRecordHandler(name = "foo", topics = "bar") + public List> foo(final List> bar) { + return null; + } + + } + + @Test + public void foo() { + final var manager = new KafkaProcessorManager(Config.getInstance()); + manager.registerHandler(new TestHandler()); + } + +} \ No newline at end of file From 5bdf47e32b19f0cfba966efe745fe903795bd0e5 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 02:25:59 +0100 Subject: [PATCH 03/26] WIP: Use properly typed interfaces instead of reflection for record processors Signed-off-by: nscuro --- .../event/kafka/processor/KafkaProcessor.java | 28 ++ .../kafka/processor/KafkaProcessorConfig.java | 85 ----- .../processor/KafkaProcessorFactory.java | 214 ++++++++++++ .../processor/KafkaProcessorHealthCheck.java | 45 --- .../processor/KafkaProcessorInitializer.java | 19 +- .../processor/KafkaProcessorManager.java | 284 ---------------- .../kafka/processor/KafkaRecordHandler.java | 24 -- ...dVulnerabilityScanResultBatchConsumer.java | 17 + .../RepositoryMetaResultConsumer.java | 207 ++++++++++++ .../VulnerabilityMirrorConsumer.java | 311 ++++++++++++++++++ .../VulnerabilityScanResultProcessor.java | 26 ++ .../api/AbstractRecordProcessingStrategy.java | 103 ++++++ .../processor/api/RecordBatchConsumer.java | 34 ++ .../api/RecordBatchProcessingStrategy.java | 89 +++++ .../processor/api/RecordBatchProcessor.java | 28 ++ .../kafka/processor/api/RecordConsumer.java | 34 ++ .../api/RecordProcessingStrategy.java | 14 + .../kafka/processor/api/RecordProcessor.java | 28 ++ .../api/SingleRecordProcessingStrategy.java | 82 +++++ .../exception/RecordProcessingException.java | 29 ++ .../RetryableRecordProcessingException.java | 29 ++ .../health/HealthCheckInitializer.java | 2 - .../processor/KafkaProcessorManagerTest.java | 27 -- .../kafka/processor/KafkaProcessorTest.java | 71 ++++ 24 files changed, 1349 insertions(+), 481 deletions(-) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java delete mode 100644 src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java create mode 100644 src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java new file mode 100644 index 000000000..7221094ff --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java @@ -0,0 +1,28 @@ +package org.dependencytrack.event.kafka.processor; + +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import org.dependencytrack.event.kafka.processor.api.RecordProcessingStrategy; + +public class KafkaProcessor implements AutoCloseable { + + private final ParallelStreamProcessor streamProcessor; + private final RecordProcessingStrategy processingStrategy; + + KafkaProcessor(final ParallelStreamProcessor streamProcessor, + final RecordProcessingStrategy processingStrategy) { + this.streamProcessor = streamProcessor; + this.processingStrategy = processingStrategy; + } + + public void start() { + // TODO: Ensure streamProcessor is subscribed to at least one topic. + processingStrategy.process(streamProcessor); + } + + @Override + public void close() { + // TODO: Drain timeout should be configurable. + streamProcessor.closeDrainFirst(); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java deleted file mode 100644 index 033f57e07..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorConfig.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; - -import java.lang.reflect.Method; -import java.util.Collection; - -public class KafkaProcessorConfig { - - private String name; - private Collection topics; - private Object handlerInstance; - private Method handlerMethod; - private ProcessingOrder ordering; - private int maxBatchSize; - private int maxConcurrency; - - public String name() { - return name; - } - - public KafkaProcessorConfig setName(String name) { - this.name = name; - return this; - } - - public Collection topics() { - return topics; - } - - public KafkaProcessorConfig setTopics(Collection topics) { - this.topics = topics; - return this; - } - - public Object handlerInstance() { - return handlerInstance; - } - - public KafkaProcessorConfig setHandlerInstance(Object handlerInstance) { - this.handlerInstance = handlerInstance; - return this; - } - - public Method handlerMethod() { - return handlerMethod; - } - - public KafkaProcessorConfig setHandlerMethod(Method handlerMethod) { - this.handlerMethod = handlerMethod; - return this; - } - - public ProcessingOrder ordering() { - return ordering; - } - - public KafkaProcessorConfig setOrdering(ProcessingOrder ordering) { - this.ordering = ordering; - return this; - } - - public int maxBatchSize() { - return maxBatchSize; - } - - public KafkaProcessorConfig setMaxBatchSize(int maxBatchSize) { - this.maxBatchSize = maxBatchSize; - return this; - } - - public int maxConcurrency() { - return maxConcurrency; - } - - public KafkaProcessorConfig setMaxConcurrency(int maxConcurrency) { - this.maxConcurrency = maxConcurrency; - return this; - } - - public boolean isBatch() { - return maxBatchSize > 1; - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java new file mode 100644 index 000000000..e2de1393e --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java @@ -0,0 +1,214 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.Config; +import alpine.common.logging.Logger; +import io.confluent.parallelconsumer.ParallelConsumerOptions; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.common.serialization.VoidDeserializer; +import org.apache.kafka.common.serialization.VoidSerializer; +import org.cyclonedx.proto.v1_4.Bom; +import org.dependencytrack.event.kafka.processor.api.RecordBatchConsumer; +import org.dependencytrack.event.kafka.processor.api.RecordBatchProcessor; +import org.dependencytrack.event.kafka.processor.api.RecordConsumer; +import org.dependencytrack.event.kafka.processor.api.RecordProcessor; +import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessingStrategy; +import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; +import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; +import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult; +import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; +import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.CLIENT_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; +import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; + +public class KafkaProcessorFactory { + + private static final Logger LOGGER = Logger.getLogger(KafkaProcessorFactory.class); + private static final Map> DESERIALIZERS = Map.ofEntries( + Map.entry(Void.TYPE.getTypeName(), new VoidDeserializer()), + Map.entry(String.class.getTypeName(), new StringDeserializer()), + Map.entry(AnalysisResult.class.getTypeName(), new KafkaProtobufDeserializer<>(AnalysisResult.parser())), + Map.entry(Bom.class.getTypeName(), new KafkaProtobufDeserializer<>(Bom.parser())), + Map.entry(ScanKey.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanKey.parser())), + Map.entry(ScanResult.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanResult.parser())) + ); + private static final Map> SERIALIZERS = Map.ofEntries( + Map.entry(Void.TYPE.getTypeName(), new VoidSerializer()), + Map.entry(String.class.getTypeName(), new StringSerializer()), + Map.entry(AnalysisResult.class.getTypeName(), new KafkaProtobufSerializer<>()), + Map.entry(Bom.class.getTypeName(), new KafkaProtobufSerializer<>()), + Map.entry(ScanKey.class.getTypeName(), new KafkaProtobufSerializer<>()), + Map.entry(ScanResult.class.getTypeName(), new KafkaProtobufSerializer<>()) + ); + + private final Config config; + + public KafkaProcessorFactory() { + this(Config.getInstance()); + } + + KafkaProcessorFactory(final Config config) { + this.config = config; + } + + public KafkaProcessor createProcessor(final RecordProcessor recordProcessor) { + final ParallelStreamProcessor streamProcessor = createStreamProcessor(recordProcessor); + final Map.Entry, Deserializer> keyValueDeserializers = keyValueDeserializers(recordProcessor.getClass()); + final Map.Entry, Serializer> keyValueSerializers = keyValueSerializers(recordProcessor.getClass()); + return new KafkaProcessor(streamProcessor, new SingleRecordProcessingStrategy<>(recordProcessor, + keyValueDeserializers.getKey(), keyValueDeserializers.getValue(), + keyValueSerializers.getKey(), keyValueSerializers.getValue())); + } + + public KafkaProcessor createBatchProcessor(final RecordBatchProcessor batchRecordProcessor) { + return null; + } + + public KafkaProcessor createConsumer(final RecordConsumer recordConsumer, + final Deserializer keyDeserializer, final Deserializer valueDeserializer) { + final ParallelStreamProcessor streamProcessor = createStreamProcessor(recordConsumer); + final var recordPollingStrategy = new SingleRecordProcessingStrategy<>(recordConsumer, + keyDeserializer, valueDeserializer, new VoidSerializer(), new VoidSerializer()); + return new KafkaProcessor(streamProcessor, recordPollingStrategy); + } + + public KafkaProcessor createBatchConsumer(final RecordBatchConsumer batchRecordConsumer) { + return null; + } + + private ParallelStreamProcessor createStreamProcessor(final Object recordProcessor) { + final var optionsBuilder = ParallelConsumerOptions.builder() + .consumer(createConsumer()); + + // TODO: How to best pass (customizable) configuration here? + + if (!isRecordConsumer(recordProcessor)) { + optionsBuilder.producer(createProducer()); + } + + return ParallelStreamProcessor.createEosStreamProcessor(optionsBuilder.build()); + } + + private Consumer createConsumer() { + final var consumerConfig = new HashMap(); + consumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); + consumerConfig.put(CLIENT_ID_CONFIG, "foo-consumer"); + consumerConfig.put(GROUP_ID_CONFIG, "foo-consumer"); + consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); + + final var psPropertyPrefix = "kafka.processor.%s.consumer".formatted("foo"); + final var psPropertyPrefixPattern = Pattern.compile(Pattern.quote("%s.".formatted(psPropertyPrefix))); + final Map psProperties = config.getPassThroughProperties(psPropertyPrefix); + for (final Map.Entry psProperty : psProperties.entrySet()) { + final String propertyName = psPropertyPrefixPattern.matcher(psProperty.getKey()).replaceFirst(""); + if (!ConsumerConfig.configNames().contains(propertyName)) { + LOGGER.warn("%s is not a known consumer config; Ignoring".formatted(propertyName)); + continue; + } + + consumerConfig.put(propertyName, psProperty.getValue()); + } + + return new KafkaConsumer<>(consumerConfig); + } + + private Producer createProducer() { + final var producerConfig = new HashMap(); + producerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); + producerConfig.put(CLIENT_ID_CONFIG, "foo-producer"); + producerConfig.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + producerConfig.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); + producerConfig.put(ENABLE_IDEMPOTENCE_CONFIG, true); + producerConfig.put(ACKS_CONFIG, "all"); + + final var psPropertyPrefix = "kafka.processor.%s.producer".formatted("foo"); + final var psPropertyPrefixPattern = Pattern.compile(Pattern.quote("%s.".formatted(psPropertyPrefix))); + final Map psProperties = config.getPassThroughProperties(psPropertyPrefix); + for (final Map.Entry psProperty : psProperties.entrySet()) { + final String propertyName = psPropertyPrefixPattern.matcher(psProperty.getKey()).replaceFirst(""); + if (!ProducerConfig.configNames().contains(propertyName)) { + LOGGER.warn("%s is not a known producer config; Ignoring".formatted(propertyName)); + continue; + } + + producerConfig.put(propertyName, psProperty.getValue()); + } + + return new KafkaProducer<>(producerConfig); + } + + private static boolean isRecordConsumer(final Object object) { + return RecordConsumer.class.isAssignableFrom(object.getClass()) + || RecordBatchConsumer.class.isAssignableFrom(object.getClass()); + } + + @SuppressWarnings("unchecked") + private static Map.Entry, Deserializer> keyValueDeserializers(final Type processorType) { + final Map.Entry keyValueTypes = consumerKeyValueTypes(processorType); + return Map.entry( + (Deserializer) DESERIALIZERS.get(keyValueTypes.getKey().getTypeName()), + (Deserializer) DESERIALIZERS.get(keyValueTypes.getValue().getTypeName()) + ); + } + + @SuppressWarnings("unchecked") + private static Map.Entry, Serializer> keyValueSerializers(final Type processorType) { + final Map.Entry keyValueTypes = producerKeyValueTypes(processorType); + return Map.entry( + (Serializer) SERIALIZERS.get(keyValueTypes.getKey().getTypeName()), + (Serializer) SERIALIZERS.get(keyValueTypes.getValue().getTypeName()) + ); + } + + private static Map.Entry consumerKeyValueTypes(final Type processorType) { + if (!(processorType instanceof final ParameterizedType paramType)) { + throw new IllegalArgumentException(""); + } + if (paramType.getActualTypeArguments().length < 2) { + throw new IllegalArgumentException(); + } + + return Map.entry(paramType.getActualTypeArguments()[0], paramType.getActualTypeArguments()[1]); + } + + private static Map.Entry producerKeyValueTypes(final Type type) { + if (!(type instanceof final ParameterizedType paramType)) { + throw new IllegalArgumentException(""); + } + if (paramType.getActualTypeArguments().length < 4) { + throw new IllegalArgumentException(); + } + + return Map.entry(paramType.getActualTypeArguments()[2], paramType.getActualTypeArguments()[3]); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java deleted file mode 100644 index b676b8e97..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorHealthCheck.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; -import io.confluent.parallelconsumer.ParallelStreamProcessor; -import org.eclipse.microprofile.health.HealthCheck; -import org.eclipse.microprofile.health.HealthCheckResponse; -import org.eclipse.microprofile.health.HealthCheckResponseBuilder; -import org.eclipse.microprofile.health.Liveness; - -import java.util.Map; - -@Liveness -public class KafkaProcessorHealthCheck implements HealthCheck { - - @Override - public HealthCheckResponse call() { - final KafkaProcessorManager processorManager = KafkaProcessorInitializer.processorManager(); - if (processorManager == null) { - return HealthCheckResponse.builder().down().build(); - } - - boolean isUp = true; - final HealthCheckResponseBuilder responseBuilder = HealthCheckResponse.named("kafka-processor"); - for (final Map.Entry> processorByName : processorManager.processors().entrySet()) { - final String processorName = processorByName.getKey(); - final ParallelStreamProcessor processor = processorByName.getValue(); - - final boolean isProcessorUp = !processor.isClosedOrFailed(); - responseBuilder.withData(processorName, isProcessorUp - ? HealthCheckResponse.Status.UP.name() - : HealthCheckResponse.Status.DOWN.name()); - - if (!isProcessorUp - && processor instanceof final ParallelEoSStreamProcessor concreteProcessor - && concreteProcessor.getFailureCause() != null) { - isUp = false; - responseBuilder.withData("%s_failure_cause".formatted(processorName), - concreteProcessor.getFailureCause().getMessage()); - } - } - - return responseBuilder.status(isUp).build(); - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java index 09f6adc3b..b0fffb70a 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java @@ -1,6 +1,5 @@ package org.dependencytrack.event.kafka.processor; -import alpine.Config; import alpine.common.logging.Logger; import javax.servlet.ServletContextEvent; @@ -9,25 +8,17 @@ public class KafkaProcessorInitializer implements ServletContextListener { private static final Logger LOGGER = Logger.getLogger(KafkaProcessorInitializer.class); - private static KafkaProcessorManager PROCESSOR_MANAGER; @Override - public void contextInitialized(ServletContextEvent sce) { - PROCESSOR_MANAGER = new KafkaProcessorManager(Config.getInstance()); - LOGGER.info("Initializing Kafka processors (instance ID: %s)".formatted(PROCESSOR_MANAGER.getInstanceId())); + public void contextInitialized(final ServletContextEvent event) { + LOGGER.info("Initializing Kafka processors"); + + final var processorFactory = new KafkaProcessorFactory(); } @Override - public void contextDestroyed(ServletContextEvent sce) { + public void contextDestroyed(final ServletContextEvent event) { LOGGER.info("Stopping Kafka processors"); - if (PROCESSOR_MANAGER != null) { - PROCESSOR_MANAGER.close(); - PROCESSOR_MANAGER = null; - } - } - - static KafkaProcessorManager processorManager() { - return PROCESSOR_MANAGER; } } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java deleted file mode 100644 index 9c134ee97..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManager.java +++ /dev/null @@ -1,284 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import alpine.Config; -import alpine.common.logging.Logger; -import alpine.common.metrics.Metrics; -import io.confluent.parallelconsumer.ParallelConsumerOptions; -import io.confluent.parallelconsumer.ParallelStreamProcessor; -import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.common.serialization.VoidDeserializer; -import org.apache.kafka.common.serialization.VoidSerializer; -import org.cyclonedx.proto.v1_4.Bom; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; -import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; -import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; - -import java.lang.reflect.Method; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static alpine.Config.AlpineKey.METRICS_ENABLED; -import static io.confluent.parallelconsumer.ParallelStreamProcessor.createEosStreamProcessor; -import static java.util.Map.entry; -import static java.util.Objects.requireNonNull; -import static org.apache.commons.lang3.reflect.MethodUtils.getMethodsListWithAnnotation; -import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.CLIENT_ID_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; -import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; - -public class KafkaProcessorManager implements AutoCloseable { - - private static final Logger LOGGER = Logger.getLogger(KafkaProcessorManager.class); - private static final Map> SERIALIZERS = Map.ofEntries( - entry("void", new VoidSerializer()), - entry(String.class.getTypeName(), new StringSerializer()), - entry(ScanKey.class.getTypeName(), new KafkaProtobufSerializer<>()), - entry(ScanResult.class.getTypeName(), new KafkaProtobufSerializer<>()) - ); - private static final Map> DESERIALIZERS = Map.ofEntries( - entry("void", new VoidDeserializer()), - entry(String.class.getTypeName(), new StringDeserializer()), - entry(Bom.class.getTypeName(), new KafkaProtobufDeserializer<>(Bom.parser())), - entry(ScanKey.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanKey.parser())), - entry(ScanResult.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanResult.parser())) - ); - - private final UUID instanceId = UUID.randomUUID(); - private final Map> processors = new HashMap<>(); - - private final Config config; - - KafkaProcessorManager(final Config config) { - this.config = config; - } - - public void registerHandler(final Object handler) { - requireNonNull(handler); - - final List processorConfigs = - getMethodsListWithAnnotation(handler.getClass(), KafkaRecordHandler.class).stream() - .map(method -> processHandlerMethod(handler, method)) - .toList(); - if (processorConfigs.isEmpty()) { - throw new IllegalArgumentException(""); - } - - for (final KafkaProcessorConfig processorConfig : processorConfigs) { - final ParallelStreamProcessor processor = - processors.computeIfAbsent(processorConfig.name(), ignored -> createProcessor(processorConfig)); - processor.poll(pollCtx -> { - - }); - } - } - - public UUID getInstanceId() { - return instanceId; - } - - public Map> processors() { - return Collections.unmodifiableMap(processors); - } - - @Override - public void close() { - processors.forEach((processorName, processor) -> { - LOGGER.info("Closing processor %s".formatted(processorName)); - processor.closeDontDrainFirst(); - }); - } - - private KafkaProcessorConfig processHandlerMethod(final Object instance, final Method method) { - final Type[] parameterTypes = method.getGenericParameterTypes(); - if (parameterTypes.length != 1) { - throw new IllegalArgumentException("Handler methods must accepts exactly one parameter, but %s is expecting %d" - .formatted(method.getName(), parameterTypes.length)); - } - - final Type parameterType = parameterTypes[0]; - if (!isConsumerRecord(parameterType) && !isConsumerRecordList(parameterType)) { - throw new IllegalArgumentException(""" - Handler methods must accept one or multiple %ss, but %s is expecting %s instead\ - """.formatted(ConsumerRecord.class.getTypeName(), method.getName(), parameterType.getTypeName())); - } - - final Type returnType = method.getGenericReturnType(); - if (!isVoid(returnType) && !isProducerRecord(returnType) && !isProducerRecordList(returnType)) { - throw new IllegalArgumentException(""" - Handler methods must return either %s, or one or multiple %ss, but %s is returning %s instead\ - """.formatted(Void.TYPE.getTypeName(), ProducerRecord.class.getTypeName(), method.getName(), returnType.getTypeName())); - } - - final Map.Entry consumerKeyValueTypes; - if (isConsumerRecord(parameterType)) { - final var paramType = (ParameterizedType) parameterType; - consumerKeyValueTypes = Map.entry(paramType.getActualTypeArguments()[0], paramType.getActualTypeArguments()[1]); - } else if (isConsumerRecordList(parameterType)) { - final var listParamType = (ParameterizedType) parameterType; - final var recordParamType = (ParameterizedType) listParamType.getActualTypeArguments()[0]; - consumerKeyValueTypes = Map.entry(recordParamType.getActualTypeArguments()[0], recordParamType.getActualTypeArguments()[1]); - } else { - throw new IllegalStateException(""); - } - - if (!DESERIALIZERS.containsKey(consumerKeyValueTypes.getKey().getTypeName())) { - throw new IllegalStateException("No deserializer known for key type %s" - .formatted(consumerKeyValueTypes.getKey().getTypeName())); - } - if (!DESERIALIZERS.containsKey(consumerKeyValueTypes.getValue().getTypeName())) { - throw new IllegalStateException("No deserializer known for value type %s" - .formatted(consumerKeyValueTypes.getValue().getTypeName())); - } - - final KafkaRecordHandler annotation = method.getAnnotation(KafkaRecordHandler.class); - return new KafkaProcessorConfig() - .setName(annotation.name()) - .setTopics(Arrays.asList(annotation.topics())) - .setHandlerInstance(instance) - .setHandlerMethod(method) - .setMaxBatchSize(annotation.maxBatchSize()) - .setMaxConcurrency(annotation.maxConcurrency()); - } - - private ParallelStreamProcessor createProcessor(final KafkaProcessorConfig processorConfig) { - final var optionsBuilder = ParallelConsumerOptions.builder() - .consumer(createConsumer(processorConfig.name())) - .maxConcurrency(processorConfig.maxConcurrency()) - .ordering(processorConfig.ordering()); - - if (processorConfig.isBatch()) { - optionsBuilder.batchSize(processorConfig.maxBatchSize()); - } - - if (config.getPropertyAsBoolean(METRICS_ENABLED)) { - optionsBuilder - .meterRegistry(Metrics.getRegistry()) - .pcInstanceTag(processorConfig.name()); - } - - final ParallelStreamProcessor processor = createEosStreamProcessor(optionsBuilder.build()); - processor.subscribe(processorConfig.topics()); - - return processor; - } - - private Consumer createConsumer(final String name) { - final var consumerConfig = new HashMap(); - consumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); - consumerConfig.put(CLIENT_ID_CONFIG, "%s-consumer-%s".formatted(name, instanceId)); - consumerConfig.put(GROUP_ID_CONFIG, name); - consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); - consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); - consumerConfig.put(AUTO_OFFSET_RESET_CONFIG, "latest"); - consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); - - final var propertyPrefix = "kafka.processor.%s.consumer".formatted(name); - final Map extraProperties = config.getPassThroughProperties(propertyPrefix); - for (final Map.Entry property : extraProperties.entrySet()) { - final String propertyName = property.getKey().replaceFirst(propertyPrefix, ""); - if (!ConsumerConfig.configNames().contains(propertyName)) { - LOGGER.warn("Provided property %s is not a known consumer config; Skipping".formatted(propertyName)); - continue; - } - - consumerConfig.put(propertyName, property.getValue()); - } - - final var consumer = new KafkaConsumer(consumerConfig); - if (config.getPropertyAsBoolean(METRICS_ENABLED)) { - new KafkaClientMetrics(consumer).bindTo(Metrics.getRegistry()); - } - - return consumer; - } - - private Producer createProducer(final String name) { - final var producerConfig = new HashMap(); - producerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); - producerConfig.put(CLIENT_ID_CONFIG, "%s-producer-%s".formatted(name, instanceId)); - producerConfig.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - producerConfig.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - - final var propertyPrefix = "kafka.processor.%s.producer".formatted(name); - final Map extraProperties = config.getPassThroughProperties(propertyPrefix); - for (final Map.Entry property : extraProperties.entrySet()) { - final String propertyName = property.getKey().replaceFirst(propertyPrefix, ""); - if (!ProducerConfig.configNames().contains(propertyName)) { - LOGGER.warn("Provided property %s is not a known producer config; Skipping".formatted(propertyName)); - continue; - } - - producerConfig.put(propertyName, property.getValue()); - } - - final var producer = new KafkaProducer(producerConfig); - if (config.getPropertyAsBoolean(METRICS_ENABLED)) { - new KafkaClientMetrics(producer).bindTo(Metrics.getRegistry()); - } - - return producer; - } - - private static boolean isConsumerRecord(final Type type) { - // ConsumerRecord - return type instanceof final ParameterizedType paramType - && ConsumerRecord.class.getTypeName().equals(paramType.getRawType().getTypeName()) - && paramType.getActualTypeArguments().length == 2; - } - - private static boolean isConsumerRecordList(final Type type) { - // List> - return type instanceof final ParameterizedType paramType - && List.class.getTypeName().equals(paramType.getRawType().getTypeName()) - && paramType.getActualTypeArguments().length == 1 - && isConsumerRecord(paramType.getActualTypeArguments()[0]); - } - - private static boolean isProducerRecord(final Type type) { - // ProducerRecord - return type instanceof final ParameterizedType paramType - && ProducerRecord.class.getTypeName().equals(paramType.getRawType().getTypeName()) - && paramType.getActualTypeArguments().length == 2; - } - - private static boolean isProducerRecordList(final Type type) { - // List> - return type instanceof final ParameterizedType paramType - && List.class.getTypeName().equals(paramType.getRawType().getTypeName()) - && paramType.getActualTypeArguments().length == 1 - && isProducerRecord(paramType.getActualTypeArguments()[0]); - } - - private static boolean isVoid(final Type type) { - return Void.TYPE.getTypeName().equals(type.getTypeName()); - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java deleted file mode 100644 index a00a77ac5..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.METHOD) -public @interface KafkaRecordHandler { - - String name(); - - String[] topics(); - - ProcessingOrder ordering() default ProcessingOrder.KEY; - - int maxBatchSize() default 1; - - int maxConcurrency() default 1; - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java new file mode 100644 index 000000000..91a982df5 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java @@ -0,0 +1,17 @@ +package org.dependencytrack.event.kafka.processor; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.event.kafka.processor.api.RecordBatchConsumer; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +import java.util.List; + +public class ProcessedVulnerabilityScanResultBatchConsumer implements RecordBatchConsumer { + + @Override + public void consume(final List> consumerRecords) throws RecordProcessingException { + // TODO: Group and aggregate by key (scan token) + // TODO: Use JDBI to batch upsert into "VULNERABILITYSCAN" table + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java new file mode 100644 index 000000000..4d376ff37 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java @@ -0,0 +1,207 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import io.micrometer.core.instrument.Timer; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.event.kafka.processor.api.RecordConsumer; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.streams.processor.RepositoryMetaResultProcessor; +import org.dependencytrack.model.FetchStatus; +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.model.RepositoryMetaComponent; +import org.dependencytrack.model.RepositoryType; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult; +import org.postgresql.util.PSQLState; + +import javax.jdo.JDODataStoreException; +import javax.jdo.PersistenceManager; +import javax.jdo.Query; +import javax.jdo.Transaction; +import java.sql.SQLException; +import java.util.Date; +import java.util.Optional; + +import static org.dependencytrack.event.kafka.componentmeta.IntegrityCheck.performIntegrityCheck; + +/** + * A {@link RecordConsumer} that ingests repository metadata {@link AnalysisResult}s. + */ +public class RepositoryMetaResultConsumer implements RecordConsumer { + + private static final Logger LOGGER = Logger.getLogger(RepositoryMetaResultProcessor.class); + private static final Timer TIMER = Timer.builder("repo_meta_result_processing") + .description("Time taken to process repository meta analysis results") + .register(Metrics.getRegistry()); + + @Override + public void consume(final ConsumerRecord record) throws RecordProcessingException { + final Timer.Sample timerSample = Timer.start(); + if (!isRecordValid(record)) { + return; + } + try (final var qm = new QueryManager()) { + synchronizeRepositoryMetadata(qm, record); + IntegrityMetaComponent integrityMetaComponent = synchronizeIntegrityMetadata(qm, record); + if (integrityMetaComponent != null) { + performIntegrityCheck(integrityMetaComponent, record.value(), qm); + } + } catch (Exception e) { + throw new RecordProcessingException("An unexpected error occurred while processing record %s".formatted(record), e); + } finally { + timerSample.stop(TIMER); + } + } + + private IntegrityMetaComponent synchronizeIntegrityMetadata(final QueryManager queryManager, final ConsumerRecord record) throws MalformedPackageURLException { + final AnalysisResult result = record.value(); + PackageURL purl = new PackageURL(result.getComponent().getPurl()); + if (result.hasIntegrityMeta()) { + return synchronizeIntegrityMetaResult(record, queryManager, purl); + } else { + LOGGER.debug("Incoming result for component with purl %s does not include component integrity info".formatted(purl)); + return null; + } + } + + private void synchronizeRepositoryMetadata(final QueryManager queryManager, final ConsumerRecord record) throws Exception { + PersistenceManager pm = queryManager.getPersistenceManager(); + final AnalysisResult result = record.value(); + PackageURL purl = new PackageURL(result.getComponent().getPurl()); + + // It is possible that the same meta info is reported for multiple components in parallel, + // causing unique constraint violations when attempting to insert into the REPOSITORY_META_COMPONENT table. + // In such cases, we can get away with simply retrying to SELECT+UPDATE or INSERT again. We'll attempt + // up to 3 times before giving up. + for (int i = 0; i < 3; i++) { + final Transaction trx = pm.currentTransaction(); + try { + RepositoryMetaComponent repositoryMetaComponentResult = createRepositoryMetaResult(record, pm, purl); + if (repositoryMetaComponentResult != null) { + trx.begin(); + pm.makePersistent(repositoryMetaComponentResult); + trx.commit(); + break; // this means that transaction was successful and we do not need to retry + } + } catch (JDODataStoreException e) { + // TODO: DataNucleus doesn't map constraint violation exceptions very well, + // so we have to depend on the exception of the underlying JDBC driver to + // tell us what happened. We currently only handle PostgreSQL, but we'll have + // to do the same for at least H2 and MSSQL. + if (ExceptionUtils.getRootCause(e) instanceof final SQLException se + && PSQLState.UNIQUE_VIOLATION.getState().equals(se.getSQLState())) { + continue; // Retry + } + + throw e; + } finally { + if (trx.isActive()) { + trx.rollback(); + } + } + } + } + + private RepositoryMetaComponent createRepositoryMetaResult(ConsumerRecord incomingAnalysisResultRecord, PersistenceManager pm, PackageURL purl) throws Exception { + final AnalysisResult result = incomingAnalysisResultRecord.value(); + if (result.hasLatestVersion()) { + try (final Query query = pm.newQuery(RepositoryMetaComponent.class)) { + query.setFilter("repositoryType == :repositoryType && namespace == :namespace && name == :name"); + query.setParameters( + RepositoryType.resolve(purl), + purl.getNamespace(), + purl.getName() + ); + RepositoryMetaComponent persistentRepoMetaComponent = query.executeUnique(); + if (persistentRepoMetaComponent == null) { + persistentRepoMetaComponent = new RepositoryMetaComponent(); + } + + if (persistentRepoMetaComponent.getLastCheck() != null + && persistentRepoMetaComponent.getLastCheck().after(new Date(incomingAnalysisResultRecord.timestamp()))) { + LOGGER.warn(""" + Received repository meta information for %s that is older\s + than what's already in the database; Discarding + """.formatted(purl)); + return null; + } + + persistentRepoMetaComponent.setRepositoryType(RepositoryType.resolve(purl)); + persistentRepoMetaComponent.setNamespace(purl.getNamespace()); + persistentRepoMetaComponent.setName(purl.getName()); + if (result.hasLatestVersion()) { + persistentRepoMetaComponent.setLatestVersion(result.getLatestVersion()); + } + if (result.hasPublished()) { + persistentRepoMetaComponent.setPublished(new Date(result.getPublished().getSeconds() * 1000)); + } + persistentRepoMetaComponent.setLastCheck(new Date(incomingAnalysisResultRecord.timestamp())); + return persistentRepoMetaComponent; + } + } else { + return null; + } + } + + private IntegrityMetaComponent synchronizeIntegrityMetaResult(final ConsumerRecord incomingAnalysisResultRecord, QueryManager queryManager, PackageURL purl) { + final AnalysisResult result = incomingAnalysisResultRecord.value(); + IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(purl.toString()); + if (persistentIntegrityMetaComponent != null && persistentIntegrityMetaComponent.getStatus() != null && persistentIntegrityMetaComponent.getStatus().equals(FetchStatus.PROCESSED)) { + LOGGER.warn(""" + Received hash information for %s that has already been processed; Discarding + """.formatted(purl)); + return persistentIntegrityMetaComponent; + } + if (persistentIntegrityMetaComponent == null) { + persistentIntegrityMetaComponent = new IntegrityMetaComponent(); + } + + if (result.getIntegrityMeta().hasMd5() || result.getIntegrityMeta().hasSha1() || result.getIntegrityMeta().hasSha256() + || result.getIntegrityMeta().hasSha512() || result.getIntegrityMeta().hasCurrentVersionLastModified()) { + Optional.ofNullable(result.getIntegrityMeta().getMd5()).ifPresent(persistentIntegrityMetaComponent::setMd5); + Optional.ofNullable(result.getIntegrityMeta().getSha1()).ifPresent(persistentIntegrityMetaComponent::setSha1); + Optional.ofNullable(result.getIntegrityMeta().getSha256()).ifPresent(persistentIntegrityMetaComponent::setSha256); + Optional.ofNullable(result.getIntegrityMeta().getSha512()).ifPresent(persistentIntegrityMetaComponent::setSha512); + persistentIntegrityMetaComponent.setPurl(result.getComponent().getPurl()); + persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl()); + persistentIntegrityMetaComponent.setPublishedAt(result.getIntegrityMeta().hasCurrentVersionLastModified() ? new Date(result.getIntegrityMeta().getCurrentVersionLastModified().getSeconds() * 1000) : null); + persistentIntegrityMetaComponent.setStatus(FetchStatus.PROCESSED); + } else { + persistentIntegrityMetaComponent.setMd5(null); + persistentIntegrityMetaComponent.setSha256(null); + persistentIntegrityMetaComponent.setSha1(null); + persistentIntegrityMetaComponent.setSha512(null); + persistentIntegrityMetaComponent.setPurl(purl.toString()); + persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl()); + persistentIntegrityMetaComponent.setStatus(FetchStatus.NOT_AVAILABLE); + } + return queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent); + } + + private static boolean isRecordValid(final ConsumerRecord record) { + final AnalysisResult result = record.value(); + if (!result.hasComponent()) { + LOGGER.warn(""" + Received repository meta information without component,\s + will not be able to correlate; Dropping + """); + return false; + } + + try { + new PackageURL(result.getComponent().getPurl()); + } catch (MalformedPackageURLException e) { + LOGGER.warn(""" + Received repository meta information with invalid PURL,\s + will not be able to correlate; Dropping + """, e); + return false; + } + return true; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java new file mode 100644 index 000000000..741693ae4 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java @@ -0,0 +1,311 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; +import io.micrometer.core.instrument.Timer; +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.cyclonedx.proto.v1_4.Bom; +import org.cyclonedx.proto.v1_4.Component; +import org.cyclonedx.proto.v1_4.VulnerabilityAffects; +import org.dependencytrack.event.kafka.processor.api.RecordConsumer; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.streams.processor.MirrorVulnerabilityProcessor; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerableSoftware; +import org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln; +import org.dependencytrack.parser.nvd.ModelConverter; +import org.dependencytrack.parser.vers.Comparator; +import org.dependencytrack.parser.vers.Constraint; +import org.dependencytrack.parser.vers.Vers; +import org.dependencytrack.parser.vers.VersException; +import org.dependencytrack.persistence.QueryManager; +import us.springett.parsers.cpe.exceptions.CpeEncodingException; +import us.springett.parsers.cpe.exceptions.CpeParsingException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * A {@link RecordConsumer} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. + */ +public class VulnerabilityMirrorConsumer implements RecordConsumer { + + private static final Logger LOGGER = Logger.getLogger(MirrorVulnerabilityProcessor.class); + private static final Timer TIMER = Timer.builder("vuln_mirror_processing") + .description("Time taken to process mirrored vulnerabilities") + .register(Metrics.getRegistry()); + + @Override + public void consume(final ConsumerRecord record) throws RecordProcessingException { + final Timer.Sample timerSample = Timer.start(); + + try (QueryManager qm = new QueryManager().withL2CacheDisabled()) { + LOGGER.debug("Synchronizing Mirrored Vulnerability : " + record.key()); + Bom bom = record.value(); + String key = record.key(); + String mirrorSource = key.substring(0, key.indexOf("/")); + Vulnerability.Source source = Vulnerability.Source.valueOf(mirrorSource); + final Vulnerability vulnerability = ModelConverterCdxToVuln.convert(qm, bom, bom.getVulnerabilities(0), false); + final List vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(vulnerability.getSource(), vulnerability.getVulnId())); + final Vulnerability synchronizedVulnerability = qm.synchronizeVulnerability(vulnerability, false); + var cycloneVuln = bom.getVulnerabilities(0); + // Alias synchronization across multiple sources is too unreliable right now. + // We can re-enable this once we have more confidence in data quality, or a better + // way of auditing reported aliases. See also: https://github.com/google/osv.dev/issues/888 +// if (!cycloneVuln.getReferencesList().isEmpty()) { +// cycloneVuln.getReferencesList().stream().forEach(reference -> { +// final String alias = reference.getId(); +// final VulnerabilityAlias vulnerabilityAlias = new VulnerabilityAlias(); +// +// // OSV will use IDs of other vulnerability databases for its +// // primary advisory ID (e.g. GHSA-45hx-wfhj-473x). We need to ensure +// // that we don't falsely report GHSA IDs as stemming from OSV. +// final Vulnerability.Source advisorySource = extractSource(cycloneVuln.getId(), cycloneVuln.getSource()); +// if (mirrorSource.equals("OSV")) { +// switch (advisorySource) { +// case NVD -> vulnerabilityAlias.setCveId(cycloneVuln.getId()); +// case GITHUB -> vulnerabilityAlias.setGhsaId(cycloneVuln.getId()); +// default -> vulnerabilityAlias.setOsvId(cycloneVuln.getId()); +// } +// } +// if (alias.startsWith("CVE") && Vulnerability.Source.NVD != advisorySource) { +// vulnerabilityAlias.setCveId(alias); +// qm.synchronizeVulnerabilityAlias(vulnerabilityAlias); +// } else if (alias.startsWith("GHSA") && Vulnerability.Source.GITHUB != advisorySource) { +// vulnerabilityAlias.setGhsaId(alias); +// qm.synchronizeVulnerabilityAlias(vulnerabilityAlias); +// } +// }); +// } + final List vsList = new ArrayList<>(); + for (final VulnerabilityAffects affect : cycloneVuln.getAffectsList()) { + final Optional component = bom.getComponentsList().stream() + .filter(c -> c.getBomRef().equals(affect.getRef())) + .findFirst(); + if (component.isEmpty()) { + LOGGER.warn("No component in the BOV for %s is matching the BOM ref \"%s\" of the affects node; Skipping" + .formatted(synchronizedVulnerability.getVulnId(), affect.getRef())); + continue; + } + + affect.getVersionsList().forEach(version -> { + if (version.hasRange()) { + final VulnerableSoftware vs = mapAffectedRangeToVulnerableSoftware(qm, + vulnerability.getVulnId(), version.getRange(), component.get().getPurl(), component.get().getCpe()); + if (vs != null) { + vsList.add(vs); + } + } + if (version.hasVersion()) { + final VulnerableSoftware vs = mapAffectedVersionToVulnerableSoftware(qm, + vulnerability.getVulnId(), version.getVersion(), component.get().getPurl(), component.get().getCpe()); + if (vs != null) { + vsList.add(vs); + } + } + }); + } + if (!vsList.isEmpty()) { + qm.persist(vsList); + qm.updateAffectedVersionAttributions(synchronizedVulnerability, vsList, source); + var reconciledVsList = qm.reconcileVulnerableSoftware(synchronizedVulnerability, vsListOld, vsList, source); + synchronizedVulnerability.setVulnerableSoftware(reconciledVsList); + } + qm.persist(synchronizedVulnerability); + // Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, Vulnerability.class)); + } catch (Exception e) { + throw new RecordProcessingException("Synchronizing vulnerability %s failed".formatted(record.key()), e); + } finally { + timerSample.stop(TIMER); + } + } + + public VulnerableSoftware mapAffectedVersionToVulnerableSoftware(final QueryManager qm, final String vulnId, + String version, String purlStr, String cpeStr) { + version = StringUtils.trimToNull(version); + cpeStr = StringUtils.trimToNull(cpeStr); + purlStr = StringUtils.trimToNull(purlStr); + if (version == null || (cpeStr == null && purlStr == null)) { + return null; + } + + var vs = new VulnerableSoftware(); + if (purlStr != null) { + final PackageURL purl; + try { + purl = new PackageURL(purlStr); + vs = qm.getVulnerableSoftwareByPurlAndVersion(purl.getType(), purl.getNamespace(), purl.getName(), version); + if (vs != null) { + return vs; + } else { + vs = new VulnerableSoftware(); + vs.setPurlType(purl.getType()); + vs.setPurlNamespace(purl.getNamespace()); + vs.setPurlName(purl.getName()); + vs.setPurl(purl.canonicalize()); + vs.setVersion(version); + } + } catch (MalformedPackageURLException e) { + LOGGER.warn("Failed to parse PURL from \"%s\" for %s; Skipping".formatted(purlStr, vulnId), e); + return null; + } + } else { + try { + vs = qm.getVulnerableSoftwareByCpe23AndVersion(cpeStr, version); + if (vs != null) { + return vs; + } else { + vs = ModelConverter.convertCpe23UriToVulnerableSoftware(cpeStr); + vs.setVersion(version); + } + } catch (CpeParsingException | CpeEncodingException e) { + LOGGER.warn("Failed to parse CPE from \"%s\" for %s; Skipping".formatted(cpeStr, vulnId), e); + return null; + } + } + vs.setVulnerable(true); + return vs; + } + + public VulnerableSoftware mapAffectedRangeToVulnerableSoftware(final QueryManager qm, final String vulnId, + String range, String purlStr, String cpeStr) { + range = StringUtils.trimToNull(range); + cpeStr = StringUtils.trimToNull(cpeStr); + purlStr = StringUtils.trimToNull(purlStr); + if (range == null || (cpeStr == null && purlStr == null)) { + return null; + } + + final Vers vers; + try { + vers = Vers.parse(range); + } catch (VersException e) { + LOGGER.warn("Failed to parse vers range from \"%s\" for %s".formatted(range, vulnId), e); + return null; + } + + if (vers.constraints().isEmpty()) { + LOGGER.debug("Vers range \"%s\" (parsed: %s) for %s does not contain any constraints; Skipping".formatted(range, vers, vulnId)); + return null; + } else if (vers.constraints().size() > 2) { + // Vers ranges can express multiple "branches", which means that this method must potentially be able to return + // multiple `VulnerableSoftware`s, not just one. For example: + // vers:tomee/>=1.0.0-beta1|<=1.7.5|>=7.0.0-M1|<=7.0.7|>=7.1.0|<=7.1.2|>=8.0.0-M1|<=8.0.1 + // would result in the following branches: + // * vers:tomee/>=1.0.0-beta1|<=1.7.5 + // * vers:tomee/>=7.0.0-M1|<=7.0.7 + // * vers:tomee/>=7.1.0|<=7.1.2 + // * vers:tomee/>=8.0.0-M1|<=8.0.1 + // + // Branches are not always pairs, for example: + // vers:npm/1.2.3|>=2.0.0|<5.0.0 + // would result in: + // * vers:npm/1.2.3 + // * vers:npm/>=2.0.0|5.0.0 + // In both cases, separate `VulnerableSoftware`s are required. + // + // Because mirror-service does not currently produce such complex ranges, we log a warning + // and skip them if we encounter them. Occurrences of this log would indicate broken parsing + // logic in mirror-service. + LOGGER.warn("Vers range \"%s\" (parsed: %s) for %s contains more than two constraints; Skipping".formatted(range, vers, vulnId)); + return null; + } + + if (vers.constraints().size() == 1 && vers.constraints().get(0).comparator() == Comparator.WILDCARD) { + // Wildcards in VulnerableSoftware can be represented via either: + // * version=*, or + // * versionStartIncluding=0 + // We choose the more explicit first option. + // + // Also, as wildcards have the potential to lead to lots of false positives, + // we want to be informed when they enter our system. So logging a warning. + LOGGER.warn("Wildcard range %s was reported for %s".formatted(vers, vulnId)); + return mapAffectedVersionToVulnerableSoftware(qm, vulnId, "*", purlStr, cpeStr); + } + + String versionStartIncluding = null; + String versionStartExcluding = null; + String versionEndIncluding = null; + String versionEndExcluding = null; + + for (final Constraint constraint : vers.constraints()) { + if (constraint.version() == null + || constraint.version().equals("0") + || constraint.version().equals("*")) { + // Semantically, ">=0" is equivalent to versionStartIncluding=null, + // and ">0" is equivalent to versionStartExcluding=null. + // + // "<0", "<=0", and "=0" can be normalized to versionStartIncluding=null. + // + // "*" is a wildcard and can only be used on its own, without any comparator. + // The Vers parsing / validation performed above will thus fail for ranges like "vers:generic/>=*". + continue; + } + + switch (constraint.comparator()) { + case GREATER_THAN -> versionStartExcluding = constraint.version(); + case GREATER_THAN_OR_EQUAL -> versionStartIncluding = constraint.version(); + case LESS_THAN_OR_EQUAL -> versionEndIncluding = constraint.version(); + case LESS_THAN -> versionEndExcluding = constraint.version(); + default -> LOGGER.warn("Encountered unexpected comparator %s in %s for %s; Skipping" + .formatted(constraint.comparator(), vers, vulnId)); + } + } + + if (versionStartIncluding == null && versionStartExcluding == null + && versionEndIncluding == null && versionEndExcluding == null) { + LOGGER.warn("Unable to assemble a version range from %s for %s".formatted(vers, vulnId)); + return null; + } + if ((versionStartIncluding != null || versionStartExcluding != null) + && (versionEndIncluding == null && versionEndExcluding == null)) { + LOGGER.warn("Skipping indefinite version range assembled from %s for %s".formatted(vers, vulnId)); + return null; + } + + VulnerableSoftware vs; + if (purlStr != null) { + final PackageURL purl; + try { + purl = new PackageURL(purlStr); + } catch (MalformedPackageURLException e) { + LOGGER.warn("Failed to parse PURL from \"%s\" for %s; Skipping".formatted(purlStr, vulnId), e); + return null; + } + vs = qm.getVulnerableSoftwareByPurl(purl.getType(), purl.getNamespace(), purl.getName(), + versionEndExcluding, versionEndIncluding, null, versionStartIncluding); + if (vs != null) { + return vs; + } + vs = new VulnerableSoftware(); + vs.setPurlType(purl.getType()); + vs.setPurlNamespace(purl.getNamespace()); + vs.setPurlName(purl.getName()); + vs.setPurl(purl.canonicalize()); + } else { + vs = qm.getVulnerableSoftwareByCpe23(cpeStr, versionEndExcluding, + versionEndIncluding, versionStartExcluding, versionStartIncluding); + if (vs != null) { + return vs; + } + try { + vs = ModelConverter.convertCpe23UriToVulnerableSoftware(cpeStr); + } catch (CpeParsingException | CpeEncodingException e) { + LOGGER.warn("Failed to parse CPE from \"%s\" for %s; Skipping".formatted(cpeStr, vulnId), e); + return null; + } + } + + vs.setVulnerable(true); + vs.setVersionStartExcluding(versionStartExcluding); + vs.setVersionStartIncluding(versionStartIncluding); + vs.setVersionEndExcluding(versionEndExcluding); + vs.setVersionEndIncluding(versionEndIncluding); + return vs; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java new file mode 100644 index 000000000..748e76a52 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java @@ -0,0 +1,26 @@ +package org.dependencytrack.event.kafka.processor; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.dependencytrack.event.kafka.processor.api.RecordProcessor; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; +import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; + +import java.util.Collections; +import java.util.List; + +/** + * TODO: This should produce ProducerRecord, where ProcessedScanResult + * is a "summary" of the processed result that can be used to update the "VULNERABILITYSCAN" table. + * The String should be the scan token, so we can avoid concurrent writes to the same "VULNERABILITYSCAN" + * row when processing the respective records. + */ +public class VulnerabilityScanResultProcessor implements RecordProcessor { + + @Override + public List> process(final ConsumerRecord record) throws RecordProcessingException { + return Collections.emptyList(); // TODO + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java new file mode 100644 index 000000000..fc0e01973 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java @@ -0,0 +1,103 @@ +package org.dependencytrack.event.kafka.processor.api; + +import alpine.common.logging.Logger; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * An abstract {@link RecordProcessingStrategy} that provides various shared functionality. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + * @param Type of the {@link ProducerRecord} key + * @param Type of the {@link ProducerRecord} value + */ +abstract class AbstractRecordProcessingStrategy implements RecordProcessingStrategy { + + private final Deserializer keyDeserializer; + private final Deserializer valueDeserializer; + private final Serializer keySerializer; + private final Serializer valueSerializer; + private final Logger logger; + + AbstractRecordProcessingStrategy(final Deserializer keyDeserializer, final Deserializer valueDeserializer, + final Serializer keySerializer, final Serializer valueSerializer) { + this.keyDeserializer = keyDeserializer; + this.valueDeserializer = valueDeserializer; + this.keySerializer = keySerializer; + this.valueSerializer = valueSerializer; + this.logger = Logger.getLogger(getClass()); + } + + /** + * @param record The {@link ConsumerRecord} to deserialize key and value of + * @return A {@link ConsumerRecord} with deserialized key and value + * @throws SerializationException When deserializing the {@link ConsumerRecord} failed + */ + ConsumerRecord deserialize(final ConsumerRecord record) { + final CK deserializedKey; + final CV deserializedValue; + try { + deserializedKey = keyDeserializer.deserialize(record.topic(), record.key()); + deserializedValue = valueDeserializer.deserialize(record.topic(), record.value()); + } catch (RuntimeException e) { + if (e instanceof SerializationException) { + throw e; + } + + throw new SerializationException(e); + } + + return new ConsumerRecord<>(record.topic(), record.partition(), record.offset(), + record.timestamp(), record.timestampType(), record.serializedKeySize(), record.serializedValueSize(), + deserializedKey, deserializedValue, record.headers(), record.leaderEpoch()); + } + + /** + * @param record The {@link ProducerRecord} to serialize key and value of + * @return A {@link ProducerRecord} with serialized key and value + * @throws SerializationException When serializing the {@link ProducerRecord} failed + */ + ProducerRecord serialize(final ProducerRecord record) { + final byte[] serializedKey; + final byte[] serializedValue; + try { + serializedKey = keySerializer.serialize(record.topic(), record.key()); + serializedValue = valueSerializer.serialize(record.topic(), record.value()); + } catch (RuntimeException e) { + if (e instanceof SerializationException) { + throw e; + } + + throw new SerializationException(e); + } + + return new ProducerRecord<>(record.topic(), record.partition(), record.timestamp(), + serializedKey, serializedValue, record.headers()); + } + + List> maybeSerializeAll(final List> records) { + if (records == null || records.isEmpty()) { + return Collections.emptyList(); + } + + final var serializedRecords = new ArrayList>(records.size()); + for (final ProducerRecord producerRecord : records) { + try { + serializedRecords.add(serialize(producerRecord)); + } catch (SerializationException e) { + logger.warn("Failed to serialize producer record %s".formatted(producerRecord), e); + } + } + + return serializedRecords; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java new file mode 100644 index 000000000..d382b1595 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java @@ -0,0 +1,34 @@ +package org.dependencytrack.event.kafka.processor.api; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +import java.util.List; + +/** + * A specialized {@link RecordBatchProcessor} that only consumes records, but does not produce any. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + */ +public interface RecordBatchConsumer extends RecordBatchProcessor { + + /** + * Consume a batch of {@link ConsumerRecord}s. + * + * @param records Batch of {@link ConsumerRecord}s to process + * @throws RecordProcessingException When consuming the batch of {@link ConsumerRecord}s failed + */ + void consume(final List> records) throws RecordProcessingException; + + /** + * {@inheritDoc} + */ + @Override + default List> process(final List> records) throws RecordProcessingException { + consume(records); + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java new file mode 100644 index 000000000..d70651403 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java @@ -0,0 +1,89 @@ +package org.dependencytrack.event.kafka.processor.api; + +import alpine.common.logging.Logger; +import io.confluent.parallelconsumer.PCRetriableException; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.confluent.parallelconsumer.PollContext; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A {@link RecordProcessingStrategy} that processes records in batches. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + * @param Type of the {@link ProducerRecord} key + * @param Type of the {@link ProducerRecord} value + */ +public class RecordBatchProcessingStrategy extends AbstractRecordProcessingStrategy { + + private static final Logger LOGGER = Logger.getLogger(RecordBatchProcessingStrategy.class); + + private final RecordBatchProcessor batchRecordProcessor; + + public RecordBatchProcessingStrategy(final RecordBatchProcessor batchRecordProcessor, + final Deserializer keyDeserializer, final Deserializer valueDeserializer, + final Serializer keySerializer, final Serializer valueSerializer) { + super(keyDeserializer, valueDeserializer, keySerializer, valueSerializer); + this.batchRecordProcessor = batchRecordProcessor; + } + + /** + * {@inheritDoc} + */ + @Override + public void process(final ParallelStreamProcessor streamProcessor) { + if (!canProduce(batchRecordProcessor)) { + streamProcessor.pollAndProduceMany(this::handlePoll); + } else { + streamProcessor.poll(this::handlePoll); + } + } + + private List> handlePoll(final PollContext pollCtx) { + final List> records = pollCtx.getConsumerRecordsFlattened(); + + final var deserializedRecords = new ArrayList>(records.size()); + for (final ConsumerRecord record : records) { + try { + deserializedRecords.add(deserialize(record)); + } catch (SerializationException e) { + // TODO: Consider supporting error handlers, e.g. to send record to DLT. + LOGGER.error("Failed to deserialize consumer record %s; Skipping".formatted(record), e); + } + } + + if (deserializedRecords.isEmpty()) { + LOGGER.warn("All of the %d records in this batch failed to be deserialized".formatted(records.size())); + return Collections.emptyList(); + } + + final List> producerRecords; + try { + producerRecords = batchRecordProcessor.process(deserializedRecords); + } catch (RetryableRecordProcessingException e) { + LOGGER.warn("Encountered retryable exception while processing %d records".formatted(deserializedRecords.size()), e); + throw new PCRetriableException(e); + } catch (RecordProcessingException | RuntimeException e) { + LOGGER.error("Encountered non-retryable exception while processing %d records; Skipping".formatted(deserializedRecords.size()), e); + // TODO: Consider supporting error handlers, e.g. to send records to DLT. + return Collections.emptyList(); // Skip records to avoid poison-pill scenario. + } + + return maybeSerializeAll(producerRecords); + } + + private static boolean canProduce(final RecordBatchProcessor processor) { + return RecordBatchConsumer.class.isAssignableFrom(processor.getClass()); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java new file mode 100644 index 000000000..c8550718d --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java @@ -0,0 +1,28 @@ +package org.dependencytrack.event.kafka.processor.api; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +import java.util.List; + +/** + * A processor of {@link ConsumerRecord} batches, capable of producing zero or more {@link ProducerRecord}s. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + * @param Type of the {@link ProducerRecord} key + * @param Type of the {@link ProducerRecord} value + */ +public interface RecordBatchProcessor { + + /** + * Process a batch of {@link ConsumerRecord}s, and produce zero or more {@link ProducerRecord}s. + * + * @param records Batch of {@link ConsumerRecord}s to process + * @return Zero or more {@link ProducerRecord}s + * @throws RecordProcessingException When processing the batch of {@link ConsumerRecord}s failed + */ + List> process(final List> records) throws RecordProcessingException; + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java new file mode 100644 index 000000000..9e9cd6339 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java @@ -0,0 +1,34 @@ +package org.dependencytrack.event.kafka.processor.api; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +import java.util.List; + +/** + * A specialized {@link RecordProcessor} that only consumes records, but does not produce any. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + */ +public interface RecordConsumer extends RecordProcessor { + + /** + * Consume a {@link ConsumerRecord}. + * + * @param record The {@link ConsumerRecord} to process + * @throws RecordProcessingException When processing the {@link ConsumerRecord} failed + */ + void consume(final ConsumerRecord record) throws RecordProcessingException; + + /** + * {@inheritDoc} + */ + @Override + default List> process(final ConsumerRecord record) throws RecordProcessingException { + consume(record); + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java new file mode 100644 index 000000000..e9934b775 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java @@ -0,0 +1,14 @@ +package org.dependencytrack.event.kafka.processor.api; + +import io.confluent.parallelconsumer.ParallelStreamProcessor; + +public interface RecordProcessingStrategy { + + /** + * Process records provided by a given {@link ParallelStreamProcessor}. + * + * @param streamProcessor The {@link ParallelStreamProcessor} to process records from + */ + void process(final ParallelStreamProcessor streamProcessor); + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java new file mode 100644 index 000000000..3ffd3e6f5 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java @@ -0,0 +1,28 @@ +package org.dependencytrack.event.kafka.processor.api; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +import java.util.List; + +/** + * A processor of individual {@link ConsumerRecord}s, capable of producing zero or more {@link ProducerRecord}s. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + * @param Type of the {@link ProducerRecord} key + * @param Type of the {@link ProducerRecord} value + */ +public interface RecordProcessor { + + /** + * Process a {@link ConsumerRecord}, and produce zero or more {@link ProducerRecord}s. + * + * @param record The {@link ConsumerRecord} to process + * @return Zero or more {@link ProducerRecord}s + * @throws RecordProcessingException When processing the {@link ConsumerRecord} failed + */ + List> process(final ConsumerRecord record) throws RecordProcessingException; + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java new file mode 100644 index 000000000..7e95796b9 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java @@ -0,0 +1,82 @@ +package org.dependencytrack.event.kafka.processor.api; + +import alpine.common.logging.Logger; +import io.confluent.parallelconsumer.PCRetriableException; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.confluent.parallelconsumer.PollContext; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; + +import java.util.Collections; +import java.util.List; + +/** + * A {@link RecordProcessingStrategy} that processes records individually. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + * @param Type of the {@link ProducerRecord} key + * @param Type of the {@link ProducerRecord} value + */ +public class SingleRecordProcessingStrategy extends AbstractRecordProcessingStrategy { + + private static final Logger LOGGER = Logger.getLogger(SingleRecordProcessingStrategy.class); + + private final RecordProcessor recordProcessor; + + public SingleRecordProcessingStrategy(final RecordProcessor recordProcessor, + final Deserializer keyDeserializer, final Deserializer valueDeserializer, + final Serializer keySerializer, final Serializer valueSerializer) { + super(keyDeserializer, valueDeserializer, keySerializer, valueSerializer); + this.recordProcessor = recordProcessor; + } + + /** + * {@inheritDoc} + */ + @Override + public void process(final ParallelStreamProcessor streamProcessor) { + if (canProduce(recordProcessor)) { + streamProcessor.pollAndProduceMany(this::handlePoll); + } else { + streamProcessor.poll(this::handlePoll); + } + } + + private List> handlePoll(final PollContext pollCtx) { + final ConsumerRecord record = pollCtx.getSingleConsumerRecord(); + + final ConsumerRecord deserializedRecord; + try { + deserializedRecord = deserialize(record); + } catch (SerializationException e) { + LOGGER.error("Failed to deserialize consumer record %s; Skipping".formatted(record), e); + // TODO: Consider supporting error handlers, e.g. to send record to DLT. + return Collections.emptyList(); // Skip record to avoid poison-pill scenario. + } + + final List> producerRecords; + try { + producerRecords = recordProcessor.process(deserializedRecord); + } catch (RetryableRecordProcessingException e) { + LOGGER.warn("Encountered retryable exception while processing %s".formatted(deserializedRecord), e); + throw new PCRetriableException(e); + } catch (RecordProcessingException | RuntimeException e) { + LOGGER.error("Encountered non-retryable exception while processing %s; Skipping".formatted(deserializedRecord), e); + // TODO: Consider supporting error handlers, e.g. to send record to DLT. + return Collections.emptyList(); // Skip record to avoid poison-pill scenario. + } + + return maybeSerializeAll(producerRecords); + } + + private static boolean canProduce(final RecordProcessor processor) { + return !RecordConsumer.class.isAssignableFrom(processor.getClass()); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java new file mode 100644 index 000000000..ba13541df --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java @@ -0,0 +1,29 @@ +package org.dependencytrack.event.kafka.processor.exception; + +/** + * An {@link Exception} indicating an error during record processing. + */ +public class RecordProcessingException extends Exception { + + /** + * {@inheritDoc} + */ + public RecordProcessingException(final String message) { + super(message); + } + + /** + * {@inheritDoc} + */ + public RecordProcessingException(final String message, final Throwable cause) { + super(message, cause); + } + + /** + * {@inheritDoc} + */ + public RecordProcessingException(final Throwable cause) { + super(cause); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java new file mode 100644 index 000000000..87af27a93 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java @@ -0,0 +1,29 @@ +package org.dependencytrack.event.kafka.processor.exception; + +/** + * A {@link RecordProcessingException} indicating a retryable error. + */ +public class RetryableRecordProcessingException extends RecordProcessingException { + + /** + * {@inheritDoc} + */ + public RetryableRecordProcessingException(final String message) { + super(message); + } + + /** + * {@inheritDoc} + */ + public RetryableRecordProcessingException(final String message, final Throwable cause) { + super(message, cause); + } + + /** + * {@inheritDoc} + */ + public RetryableRecordProcessingException(final Throwable cause) { + super(cause); + } + +} diff --git a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java index 1f6cbc232..60ad5dd04 100644 --- a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java +++ b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java @@ -24,7 +24,6 @@ import alpine.server.health.checks.DatabaseHealthCheck; import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; -import org.dependencytrack.event.kafka.processor.KafkaProcessorHealthCheck; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -38,7 +37,6 @@ public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Registering health checks"); HealthCheckRegistry.getInstance().register("database", new DatabaseHealthCheck()); HealthCheckRegistry.getInstance().register("kafka-streams", new KafkaStreamsHealthCheck()); - HealthCheckRegistry.getInstance().register("kafka-processor", new KafkaProcessorHealthCheck()); // TODO: Move this to its own initializer if it turns out to be useful LOGGER.info("Registering extra process metrics"); diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java deleted file mode 100644 index 47e7a4062..000000000 --- a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorManagerTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import alpine.Config; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.junit.Test; - -import java.util.List; - -public class KafkaProcessorManagerTest { - - public static class TestHandler { - - @KafkaRecordHandler(name = "foo", topics = "bar") - public List> foo(final List> bar) { - return null; - } - - } - - @Test - public void foo() { - final var manager = new KafkaProcessorManager(Config.getInstance()); - manager.registerHandler(new TestHandler()); - } - -} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java new file mode 100644 index 000000000..e83f7d7f2 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java @@ -0,0 +1,71 @@ +package org.dependencytrack.event.kafka.processor; + + +import net.mguenther.kafka.junit.ExternalKafkaCluster; +import net.mguenther.kafka.junit.KeyValue; +import net.mguenther.kafka.junit.ReadKeyValues; +import net.mguenther.kafka.junit.SendKeyValues; +import net.mguenther.kafka.junit.TopicConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.BooleanDeserializer; +import org.apache.kafka.common.serialization.IntegerDeserializer; +import org.apache.kafka.common.serialization.LongSerializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.dependencytrack.event.kafka.processor.api.RecordProcessor; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.testcontainers.redpanda.RedpandaContainer; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; +import java.util.List; + +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +public class KafkaProcessorTest { + + @Rule + public RedpandaContainer container = new RedpandaContainer(DockerImageName + .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); + + ExternalKafkaCluster kafka; + + @Before + public void setUp() { + kafka = ExternalKafkaCluster.at(container.getBootstrapServers()); + } + + @Test + public void testSingleRecordProcessor() throws Exception { + kafka.createTopic(TopicConfig.withName("input")); + kafka.createTopic(TopicConfig.withName("output")); + + final RecordProcessor recordProcessor = + record -> List.of(new ProducerRecord<>("output", 2, true)); + + final var processorFactory = new KafkaProcessorFactory(); + try (final KafkaProcessor processor = processorFactory.createProcessor(recordProcessor)) { + processor.start(); + + kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", 123L))) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName())); + } + + await() + .atMost(Duration.ofSeconds(5)) + .untilAsserted(() -> { + final List> records = kafka.read(ReadKeyValues.from("output", Integer.class, Boolean.class) + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class.getName()) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BooleanDeserializer.class.getName())); + + assertThat(records).hasSize(1); + }); + } + +} \ No newline at end of file From 317b19fcca0898d16671cf424c63fc08d44862b4 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 16:10:27 +0100 Subject: [PATCH 04/26] WIP: Simplify processor APIs Do not handle producing, processors can use the KafkaEventDispatcher in case they need to produce records. Processors may need to produce records of different key-value types (e.g. results and notifications), generics don't really work well in that case. Signed-off-by: nscuro --- .../event/kafka/processor/KafkaProcessor.java | 28 --- .../processor/KafkaProcessorFactory.java | 214 ------------------ .../processor/KafkaProcessorInitializer.java | 24 -- .../KafkaRecordProcessorInitializer.java | 31 +++ ...ssedVulnerabilityScanResultProcessor.java} | 6 +- ...ava => RepositoryMetaResultProcessor.java} | 11 +- ...java => VulnerabilityMirrorProcessor.java} | 8 +- .../VulnerabilityScanResultProcessor.java | 12 +- .../api/AbstractRecordProcessingStrategy.java | 84 ++----- .../api/BatchRecordProcessingStrategy.java | 67 ++++++ .../processor/api/BatchRecordProcessor.java | 24 ++ .../processor/api/RecordBatchConsumer.java | 34 --- .../api/RecordBatchProcessingStrategy.java | 89 -------- .../processor/api/RecordBatchProcessor.java | 28 --- .../kafka/processor/api/RecordConsumer.java | 34 --- .../api/RecordProcessingStrategy.java | 10 +- .../kafka/processor/api/RecordProcessor.java | 28 --- .../processor/api/RecordProcessorManager.java | 166 ++++++++++++++ .../api/SingleRecordProcessingStrategy.java | 50 ++-- .../processor/api/SingleRecordProcessor.java | 22 ++ .../kafka/processor/KafkaProcessorTest.java | 29 +-- .../api/RecordProcessorManagerTest.java | 112 +++++++++ 22 files changed, 481 insertions(+), 630 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java rename src/main/java/org/dependencytrack/event/kafka/processor/{ProcessedVulnerabilityScanResultBatchConsumer.java => ProcessedVulnerabilityScanResultProcessor.java} (63%) rename src/main/java/org/dependencytrack/event/kafka/processor/{RepositoryMetaResultConsumer.java => RepositoryMetaResultProcessor.java} (95%) rename src/main/java/org/dependencytrack/event/kafka/processor/{VulnerabilityMirrorConsumer.java => VulnerabilityMirrorProcessor.java} (97%) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java create mode 100644 src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java deleted file mode 100644 index 7221094ff..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessor.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import io.confluent.parallelconsumer.ParallelStreamProcessor; -import org.dependencytrack.event.kafka.processor.api.RecordProcessingStrategy; - -public class KafkaProcessor implements AutoCloseable { - - private final ParallelStreamProcessor streamProcessor; - private final RecordProcessingStrategy processingStrategy; - - KafkaProcessor(final ParallelStreamProcessor streamProcessor, - final RecordProcessingStrategy processingStrategy) { - this.streamProcessor = streamProcessor; - this.processingStrategy = processingStrategy; - } - - public void start() { - // TODO: Ensure streamProcessor is subscribed to at least one topic. - processingStrategy.process(streamProcessor); - } - - @Override - public void close() { - // TODO: Drain timeout should be configurable. - streamProcessor.closeDrainFirst(); - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java deleted file mode 100644 index e2de1393e..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorFactory.java +++ /dev/null @@ -1,214 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import alpine.Config; -import alpine.common.logging.Logger; -import io.confluent.parallelconsumer.ParallelConsumerOptions; -import io.confluent.parallelconsumer.ParallelStreamProcessor; -import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.common.serialization.VoidDeserializer; -import org.apache.kafka.common.serialization.VoidSerializer; -import org.cyclonedx.proto.v1_4.Bom; -import org.dependencytrack.event.kafka.processor.api.RecordBatchConsumer; -import org.dependencytrack.event.kafka.processor.api.RecordBatchProcessor; -import org.dependencytrack.event.kafka.processor.api.RecordConsumer; -import org.dependencytrack.event.kafka.processor.api.RecordProcessor; -import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessingStrategy; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; -import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult; -import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; -import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; - -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.CommonClientConfigs.CLIENT_ID_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.ACKS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; -import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; - -public class KafkaProcessorFactory { - - private static final Logger LOGGER = Logger.getLogger(KafkaProcessorFactory.class); - private static final Map> DESERIALIZERS = Map.ofEntries( - Map.entry(Void.TYPE.getTypeName(), new VoidDeserializer()), - Map.entry(String.class.getTypeName(), new StringDeserializer()), - Map.entry(AnalysisResult.class.getTypeName(), new KafkaProtobufDeserializer<>(AnalysisResult.parser())), - Map.entry(Bom.class.getTypeName(), new KafkaProtobufDeserializer<>(Bom.parser())), - Map.entry(ScanKey.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanKey.parser())), - Map.entry(ScanResult.class.getTypeName(), new KafkaProtobufDeserializer<>(ScanResult.parser())) - ); - private static final Map> SERIALIZERS = Map.ofEntries( - Map.entry(Void.TYPE.getTypeName(), new VoidSerializer()), - Map.entry(String.class.getTypeName(), new StringSerializer()), - Map.entry(AnalysisResult.class.getTypeName(), new KafkaProtobufSerializer<>()), - Map.entry(Bom.class.getTypeName(), new KafkaProtobufSerializer<>()), - Map.entry(ScanKey.class.getTypeName(), new KafkaProtobufSerializer<>()), - Map.entry(ScanResult.class.getTypeName(), new KafkaProtobufSerializer<>()) - ); - - private final Config config; - - public KafkaProcessorFactory() { - this(Config.getInstance()); - } - - KafkaProcessorFactory(final Config config) { - this.config = config; - } - - public KafkaProcessor createProcessor(final RecordProcessor recordProcessor) { - final ParallelStreamProcessor streamProcessor = createStreamProcessor(recordProcessor); - final Map.Entry, Deserializer> keyValueDeserializers = keyValueDeserializers(recordProcessor.getClass()); - final Map.Entry, Serializer> keyValueSerializers = keyValueSerializers(recordProcessor.getClass()); - return new KafkaProcessor(streamProcessor, new SingleRecordProcessingStrategy<>(recordProcessor, - keyValueDeserializers.getKey(), keyValueDeserializers.getValue(), - keyValueSerializers.getKey(), keyValueSerializers.getValue())); - } - - public KafkaProcessor createBatchProcessor(final RecordBatchProcessor batchRecordProcessor) { - return null; - } - - public KafkaProcessor createConsumer(final RecordConsumer recordConsumer, - final Deserializer keyDeserializer, final Deserializer valueDeserializer) { - final ParallelStreamProcessor streamProcessor = createStreamProcessor(recordConsumer); - final var recordPollingStrategy = new SingleRecordProcessingStrategy<>(recordConsumer, - keyDeserializer, valueDeserializer, new VoidSerializer(), new VoidSerializer()); - return new KafkaProcessor(streamProcessor, recordPollingStrategy); - } - - public KafkaProcessor createBatchConsumer(final RecordBatchConsumer batchRecordConsumer) { - return null; - } - - private ParallelStreamProcessor createStreamProcessor(final Object recordProcessor) { - final var optionsBuilder = ParallelConsumerOptions.builder() - .consumer(createConsumer()); - - // TODO: How to best pass (customizable) configuration here? - - if (!isRecordConsumer(recordProcessor)) { - optionsBuilder.producer(createProducer()); - } - - return ParallelStreamProcessor.createEosStreamProcessor(optionsBuilder.build()); - } - - private Consumer createConsumer() { - final var consumerConfig = new HashMap(); - consumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); - consumerConfig.put(CLIENT_ID_CONFIG, "foo-consumer"); - consumerConfig.put(GROUP_ID_CONFIG, "foo-consumer"); - consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); - consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); - consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); - - final var psPropertyPrefix = "kafka.processor.%s.consumer".formatted("foo"); - final var psPropertyPrefixPattern = Pattern.compile(Pattern.quote("%s.".formatted(psPropertyPrefix))); - final Map psProperties = config.getPassThroughProperties(psPropertyPrefix); - for (final Map.Entry psProperty : psProperties.entrySet()) { - final String propertyName = psPropertyPrefixPattern.matcher(psProperty.getKey()).replaceFirst(""); - if (!ConsumerConfig.configNames().contains(propertyName)) { - LOGGER.warn("%s is not a known consumer config; Ignoring".formatted(propertyName)); - continue; - } - - consumerConfig.put(propertyName, psProperty.getValue()); - } - - return new KafkaConsumer<>(consumerConfig); - } - - private Producer createProducer() { - final var producerConfig = new HashMap(); - producerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); - producerConfig.put(CLIENT_ID_CONFIG, "foo-producer"); - producerConfig.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - producerConfig.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); - producerConfig.put(ENABLE_IDEMPOTENCE_CONFIG, true); - producerConfig.put(ACKS_CONFIG, "all"); - - final var psPropertyPrefix = "kafka.processor.%s.producer".formatted("foo"); - final var psPropertyPrefixPattern = Pattern.compile(Pattern.quote("%s.".formatted(psPropertyPrefix))); - final Map psProperties = config.getPassThroughProperties(psPropertyPrefix); - for (final Map.Entry psProperty : psProperties.entrySet()) { - final String propertyName = psPropertyPrefixPattern.matcher(psProperty.getKey()).replaceFirst(""); - if (!ProducerConfig.configNames().contains(propertyName)) { - LOGGER.warn("%s is not a known producer config; Ignoring".formatted(propertyName)); - continue; - } - - producerConfig.put(propertyName, psProperty.getValue()); - } - - return new KafkaProducer<>(producerConfig); - } - - private static boolean isRecordConsumer(final Object object) { - return RecordConsumer.class.isAssignableFrom(object.getClass()) - || RecordBatchConsumer.class.isAssignableFrom(object.getClass()); - } - - @SuppressWarnings("unchecked") - private static Map.Entry, Deserializer> keyValueDeserializers(final Type processorType) { - final Map.Entry keyValueTypes = consumerKeyValueTypes(processorType); - return Map.entry( - (Deserializer) DESERIALIZERS.get(keyValueTypes.getKey().getTypeName()), - (Deserializer) DESERIALIZERS.get(keyValueTypes.getValue().getTypeName()) - ); - } - - @SuppressWarnings("unchecked") - private static Map.Entry, Serializer> keyValueSerializers(final Type processorType) { - final Map.Entry keyValueTypes = producerKeyValueTypes(processorType); - return Map.entry( - (Serializer) SERIALIZERS.get(keyValueTypes.getKey().getTypeName()), - (Serializer) SERIALIZERS.get(keyValueTypes.getValue().getTypeName()) - ); - } - - private static Map.Entry consumerKeyValueTypes(final Type processorType) { - if (!(processorType instanceof final ParameterizedType paramType)) { - throw new IllegalArgumentException(""); - } - if (paramType.getActualTypeArguments().length < 2) { - throw new IllegalArgumentException(); - } - - return Map.entry(paramType.getActualTypeArguments()[0], paramType.getActualTypeArguments()[1]); - } - - private static Map.Entry producerKeyValueTypes(final Type type) { - if (!(type instanceof final ParameterizedType paramType)) { - throw new IllegalArgumentException(""); - } - if (paramType.getActualTypeArguments().length < 4) { - throw new IllegalArgumentException(); - } - - return Map.entry(paramType.getActualTypeArguments()[2], paramType.getActualTypeArguments()[3]); - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java deleted file mode 100644 index b0fffb70a..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorInitializer.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import alpine.common.logging.Logger; - -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; - -public class KafkaProcessorInitializer implements ServletContextListener { - - private static final Logger LOGGER = Logger.getLogger(KafkaProcessorInitializer.class); - - @Override - public void contextInitialized(final ServletContextEvent event) { - LOGGER.info("Initializing Kafka processors"); - - final var processorFactory = new KafkaProcessorFactory(); - } - - @Override - public void contextDestroyed(final ServletContextEvent event) { - LOGGER.info("Stopping Kafka processors"); - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java new file mode 100644 index 000000000..ebcfaf229 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java @@ -0,0 +1,31 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.common.logging.Logger; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.event.kafka.processor.api.RecordProcessorManager; + +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +public class KafkaRecordProcessorInitializer implements ServletContextListener { + + private static final Logger LOGGER = Logger.getLogger(KafkaRecordProcessorInitializer.class); + + private final RecordProcessorManager processorManager = new RecordProcessorManager(); + + @Override + public void contextInitialized(final ServletContextEvent event) { + LOGGER.info("Initializing Kafka processors"); + + processorManager.register("vuln-mirror", new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); + processorManager.register("repo-meta-result", new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); + processorManager.startAll(); + } + + @Override + public void contextDestroyed(final ServletContextEvent event) { + LOGGER.info("Stopping Kafka processors"); + processorManager.close(); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java similarity index 63% rename from src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java rename to src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java index 91a982df5..3ca31c163 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultBatchConsumer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java @@ -1,15 +1,15 @@ package org.dependencytrack.event.kafka.processor; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.api.RecordBatchConsumer; +import org.dependencytrack.event.kafka.processor.api.BatchRecordProcessor; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; import java.util.List; -public class ProcessedVulnerabilityScanResultBatchConsumer implements RecordBatchConsumer { +public class ProcessedVulnerabilityScanResultProcessor implements BatchRecordProcessor { @Override - public void consume(final List> consumerRecords) throws RecordProcessingException { + public void process(final List> consumerRecords) throws RecordProcessingException { // TODO: Group and aggregate by key (scan token) // TODO: Use JDBI to batch upsert into "VULNERABILITYSCAN" table } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java similarity index 95% rename from src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java rename to src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index 4d376ff37..64909b0c3 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultConsumer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -7,9 +7,8 @@ import io.micrometer.core.instrument.Timer; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.api.RecordConsumer; +import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; -import org.dependencytrack.event.kafka.streams.processor.RepositoryMetaResultProcessor; import org.dependencytrack.model.FetchStatus; import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.RepositoryMetaComponent; @@ -29,17 +28,17 @@ import static org.dependencytrack.event.kafka.componentmeta.IntegrityCheck.performIntegrityCheck; /** - * A {@link RecordConsumer} that ingests repository metadata {@link AnalysisResult}s. + * A {@link SingleRecordProcessor} that ingests repository metadata {@link AnalysisResult}s. */ -public class RepositoryMetaResultConsumer implements RecordConsumer { +public class RepositoryMetaResultProcessor implements SingleRecordProcessor { - private static final Logger LOGGER = Logger.getLogger(RepositoryMetaResultProcessor.class); + private static final Logger LOGGER = Logger.getLogger(org.dependencytrack.event.kafka.streams.processor.RepositoryMetaResultProcessor.class); private static final Timer TIMER = Timer.builder("repo_meta_result_processing") .description("Time taken to process repository meta analysis results") .register(Metrics.getRegistry()); @Override - public void consume(final ConsumerRecord record) throws RecordProcessingException { + public void process(final ConsumerRecord record) throws RecordProcessingException { final Timer.Sample timerSample = Timer.start(); if (!isRecordValid(record)) { return; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java similarity index 97% rename from src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java rename to src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java index 741693ae4..d6dbabc9c 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorConsumer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java @@ -10,7 +10,7 @@ import org.cyclonedx.proto.v1_4.Bom; import org.cyclonedx.proto.v1_4.Component; import org.cyclonedx.proto.v1_4.VulnerabilityAffects; -import org.dependencytrack.event.kafka.processor.api.RecordConsumer; +import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; import org.dependencytrack.event.kafka.streams.processor.MirrorVulnerabilityProcessor; import org.dependencytrack.model.Vulnerability; @@ -30,9 +30,9 @@ import java.util.Optional; /** - * A {@link RecordConsumer} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. + * A {@link SingleRecordProcessor} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. */ -public class VulnerabilityMirrorConsumer implements RecordConsumer { +public class VulnerabilityMirrorProcessor implements SingleRecordProcessor { private static final Logger LOGGER = Logger.getLogger(MirrorVulnerabilityProcessor.class); private static final Timer TIMER = Timer.builder("vuln_mirror_processing") @@ -40,7 +40,7 @@ public class VulnerabilityMirrorConsumer implements RecordConsumer .register(Metrics.getRegistry()); @Override - public void consume(final ConsumerRecord record) throws RecordProcessingException { + public void process(final ConsumerRecord record) throws RecordProcessingException { final Timer.Sample timerSample = Timer.start(); try (QueryManager qm = new QueryManager().withL2CacheDisabled()) { diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java index 748e76a52..f25183bae 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java @@ -1,26 +1,22 @@ package org.dependencytrack.event.kafka.processor; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.dependencytrack.event.kafka.processor.api.RecordProcessor; +import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; -import java.util.Collections; -import java.util.List; - /** * TODO: This should produce ProducerRecord, where ProcessedScanResult * is a "summary" of the processed result that can be used to update the "VULNERABILITYSCAN" table. * The String should be the scan token, so we can avoid concurrent writes to the same "VULNERABILITYSCAN" * row when processing the respective records. */ -public class VulnerabilityScanResultProcessor implements RecordProcessor { +public class VulnerabilityScanResultProcessor implements SingleRecordProcessor { @Override - public List> process(final ConsumerRecord record) throws RecordProcessingException { - return Collections.emptyList(); // TODO + public void process(final ConsumerRecord record) throws RecordProcessingException { + // TODO } } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java index fc0e01973..da1cbfaec 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java @@ -1,39 +1,23 @@ package org.dependencytrack.event.kafka.processor.api; -import alpine.common.logging.Logger; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import org.apache.kafka.common.serialization.Serde; /** * An abstract {@link RecordProcessingStrategy} that provides various shared functionality. * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - * @param Type of the {@link ProducerRecord} key - * @param Type of the {@link ProducerRecord} value + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value */ -abstract class AbstractRecordProcessingStrategy implements RecordProcessingStrategy { +abstract class AbstractRecordProcessingStrategy implements RecordProcessingStrategy { - private final Deserializer keyDeserializer; - private final Deserializer valueDeserializer; - private final Serializer keySerializer; - private final Serializer valueSerializer; - private final Logger logger; + private final Serde keySerde; + private final Serde valueSerde; - AbstractRecordProcessingStrategy(final Deserializer keyDeserializer, final Deserializer valueDeserializer, - final Serializer keySerializer, final Serializer valueSerializer) { - this.keyDeserializer = keyDeserializer; - this.valueDeserializer = valueDeserializer; - this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; - this.logger = Logger.getLogger(getClass()); + AbstractRecordProcessingStrategy(final Serde keySerde, final Serde valueSerde) { + this.keySerde = keySerde; + this.valueSerde = valueSerde; } /** @@ -41,12 +25,12 @@ abstract class AbstractRecordProcessingStrategy implements Recor * @return A {@link ConsumerRecord} with deserialized key and value * @throws SerializationException When deserializing the {@link ConsumerRecord} failed */ - ConsumerRecord deserialize(final ConsumerRecord record) { - final CK deserializedKey; - final CV deserializedValue; + ConsumerRecord deserialize(final ConsumerRecord record) { + final K deserializedKey; + final V deserializedValue; try { - deserializedKey = keyDeserializer.deserialize(record.topic(), record.key()); - deserializedValue = valueDeserializer.deserialize(record.topic(), record.value()); + deserializedKey = keySerde.deserializer().deserialize(record.topic(), record.key()); + deserializedValue = valueSerde.deserializer().deserialize(record.topic(), record.value()); } catch (RuntimeException e) { if (e instanceof SerializationException) { throw e; @@ -60,44 +44,4 @@ ConsumerRecord deserialize(final ConsumerRecord record) deserializedKey, deserializedValue, record.headers(), record.leaderEpoch()); } - /** - * @param record The {@link ProducerRecord} to serialize key and value of - * @return A {@link ProducerRecord} with serialized key and value - * @throws SerializationException When serializing the {@link ProducerRecord} failed - */ - ProducerRecord serialize(final ProducerRecord record) { - final byte[] serializedKey; - final byte[] serializedValue; - try { - serializedKey = keySerializer.serialize(record.topic(), record.key()); - serializedValue = valueSerializer.serialize(record.topic(), record.value()); - } catch (RuntimeException e) { - if (e instanceof SerializationException) { - throw e; - } - - throw new SerializationException(e); - } - - return new ProducerRecord<>(record.topic(), record.partition(), record.timestamp(), - serializedKey, serializedValue, record.headers()); - } - - List> maybeSerializeAll(final List> records) { - if (records == null || records.isEmpty()) { - return Collections.emptyList(); - } - - final var serializedRecords = new ArrayList>(records.size()); - for (final ProducerRecord producerRecord : records) { - try { - serializedRecords.add(serialize(producerRecord)); - } catch (SerializationException e) { - logger.warn("Failed to serialize producer record %s".formatted(producerRecord), e); - } - } - - return serializedRecords; - } - } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java new file mode 100644 index 000000000..6034ae84c --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java @@ -0,0 +1,67 @@ +package org.dependencytrack.event.kafka.processor.api; + +import alpine.common.logging.Logger; +import io.confluent.parallelconsumer.PCRetriableException; +import io.confluent.parallelconsumer.PollContext; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Serde; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; + +import java.util.ArrayList; +import java.util.List; + +/** + * A {@link RecordProcessingStrategy} that processes records in batches. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + */ +class BatchRecordProcessingStrategy extends AbstractRecordProcessingStrategy { + + private static final Logger LOGGER = Logger.getLogger(BatchRecordProcessingStrategy.class); + + private final BatchRecordProcessor batchConsumer; + + BatchRecordProcessingStrategy(final BatchRecordProcessor batchConsumer, + final Serde keySerde, final Serde valueSerde) { + super(keySerde, valueSerde); + this.batchConsumer = batchConsumer; + } + + /** + * {@inheritDoc} + */ + @Override + public void handlePoll(final PollContext pollCtx) { + final List> records = pollCtx.getConsumerRecordsFlattened(); + + final var deserializedRecords = new ArrayList>(records.size()); + for (final ConsumerRecord record : records) { + try { + deserializedRecords.add(deserialize(record)); + } catch (SerializationException e) { + // TODO: Consider supporting error handlers, e.g. to send record to DLT. + LOGGER.error("Failed to deserialize consumer record %s; Skipping".formatted(record), e); + } + } + + if (deserializedRecords.isEmpty()) { + LOGGER.warn("All of the %d records in this batch failed to be deserialized".formatted(records.size())); + return; + } + + try { + batchConsumer.process(deserializedRecords); + } catch (RetryableRecordProcessingException e) { + LOGGER.warn("Encountered retryable exception while processing %d records".formatted(deserializedRecords.size()), e); + throw new PCRetriableException(e); + } catch (RecordProcessingException | RuntimeException e) { + LOGGER.error("Encountered non-retryable exception while processing %d records; Skipping".formatted(deserializedRecords.size()), e); + // TODO: Consider supporting error handlers, e.g. to send records to DLT. + // Skip records to avoid poison-pill scenario. + } + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java new file mode 100644 index 000000000..d9707d593 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java @@ -0,0 +1,24 @@ +package org.dependencytrack.event.kafka.processor.api; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +import java.util.List; + +/** + * A processor of {@link ConsumerRecord} batches. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + */ +public interface BatchRecordProcessor { + + /** + * Process a batch of {@link ConsumerRecord}s. + * + * @param records Batch of {@link ConsumerRecord}s to process + * @throws RecordProcessingException When consuming the batch of {@link ConsumerRecord}s failed + */ + void process(final List> records) throws RecordProcessingException; + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java deleted file mode 100644 index d382b1595..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchConsumer.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.dependencytrack.event.kafka.processor.api; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; - -import java.util.List; - -/** - * A specialized {@link RecordBatchProcessor} that only consumes records, but does not produce any. - * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - */ -public interface RecordBatchConsumer extends RecordBatchProcessor { - - /** - * Consume a batch of {@link ConsumerRecord}s. - * - * @param records Batch of {@link ConsumerRecord}s to process - * @throws RecordProcessingException When consuming the batch of {@link ConsumerRecord}s failed - */ - void consume(final List> records) throws RecordProcessingException; - - /** - * {@inheritDoc} - */ - @Override - default List> process(final List> records) throws RecordProcessingException { - consume(records); - return null; - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java deleted file mode 100644 index d70651403..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessingStrategy.java +++ /dev/null @@ -1,89 +0,0 @@ -package org.dependencytrack.event.kafka.processor.api; - -import alpine.common.logging.Logger; -import io.confluent.parallelconsumer.PCRetriableException; -import io.confluent.parallelconsumer.ParallelStreamProcessor; -import io.confluent.parallelconsumer.PollContext; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; -import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * A {@link RecordProcessingStrategy} that processes records in batches. - * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - * @param Type of the {@link ProducerRecord} key - * @param Type of the {@link ProducerRecord} value - */ -public class RecordBatchProcessingStrategy extends AbstractRecordProcessingStrategy { - - private static final Logger LOGGER = Logger.getLogger(RecordBatchProcessingStrategy.class); - - private final RecordBatchProcessor batchRecordProcessor; - - public RecordBatchProcessingStrategy(final RecordBatchProcessor batchRecordProcessor, - final Deserializer keyDeserializer, final Deserializer valueDeserializer, - final Serializer keySerializer, final Serializer valueSerializer) { - super(keyDeserializer, valueDeserializer, keySerializer, valueSerializer); - this.batchRecordProcessor = batchRecordProcessor; - } - - /** - * {@inheritDoc} - */ - @Override - public void process(final ParallelStreamProcessor streamProcessor) { - if (!canProduce(batchRecordProcessor)) { - streamProcessor.pollAndProduceMany(this::handlePoll); - } else { - streamProcessor.poll(this::handlePoll); - } - } - - private List> handlePoll(final PollContext pollCtx) { - final List> records = pollCtx.getConsumerRecordsFlattened(); - - final var deserializedRecords = new ArrayList>(records.size()); - for (final ConsumerRecord record : records) { - try { - deserializedRecords.add(deserialize(record)); - } catch (SerializationException e) { - // TODO: Consider supporting error handlers, e.g. to send record to DLT. - LOGGER.error("Failed to deserialize consumer record %s; Skipping".formatted(record), e); - } - } - - if (deserializedRecords.isEmpty()) { - LOGGER.warn("All of the %d records in this batch failed to be deserialized".formatted(records.size())); - return Collections.emptyList(); - } - - final List> producerRecords; - try { - producerRecords = batchRecordProcessor.process(deserializedRecords); - } catch (RetryableRecordProcessingException e) { - LOGGER.warn("Encountered retryable exception while processing %d records".formatted(deserializedRecords.size()), e); - throw new PCRetriableException(e); - } catch (RecordProcessingException | RuntimeException e) { - LOGGER.error("Encountered non-retryable exception while processing %d records; Skipping".formatted(deserializedRecords.size()), e); - // TODO: Consider supporting error handlers, e.g. to send records to DLT. - return Collections.emptyList(); // Skip records to avoid poison-pill scenario. - } - - return maybeSerializeAll(producerRecords); - } - - private static boolean canProduce(final RecordBatchProcessor processor) { - return RecordBatchConsumer.class.isAssignableFrom(processor.getClass()); - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java deleted file mode 100644 index c8550718d..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordBatchProcessor.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.dependencytrack.event.kafka.processor.api; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; - -import java.util.List; - -/** - * A processor of {@link ConsumerRecord} batches, capable of producing zero or more {@link ProducerRecord}s. - * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - * @param Type of the {@link ProducerRecord} key - * @param Type of the {@link ProducerRecord} value - */ -public interface RecordBatchProcessor { - - /** - * Process a batch of {@link ConsumerRecord}s, and produce zero or more {@link ProducerRecord}s. - * - * @param records Batch of {@link ConsumerRecord}s to process - * @return Zero or more {@link ProducerRecord}s - * @throws RecordProcessingException When processing the batch of {@link ConsumerRecord}s failed - */ - List> process(final List> records) throws RecordProcessingException; - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java deleted file mode 100644 index 9e9cd6339..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordConsumer.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.dependencytrack.event.kafka.processor.api; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; - -import java.util.List; - -/** - * A specialized {@link RecordProcessor} that only consumes records, but does not produce any. - * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - */ -public interface RecordConsumer extends RecordProcessor { - - /** - * Consume a {@link ConsumerRecord}. - * - * @param record The {@link ConsumerRecord} to process - * @throws RecordProcessingException When processing the {@link ConsumerRecord} failed - */ - void consume(final ConsumerRecord record) throws RecordProcessingException; - - /** - * {@inheritDoc} - */ - @Override - default List> process(final ConsumerRecord record) throws RecordProcessingException { - consume(record); - return null; - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java index e9934b775..acd358601 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java @@ -1,14 +1,14 @@ package org.dependencytrack.event.kafka.processor.api; -import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.confluent.parallelconsumer.PollContext; -public interface RecordProcessingStrategy { +interface RecordProcessingStrategy { /** - * Process records provided by a given {@link ParallelStreamProcessor}. + * Handle the result of a consumer poll. * - * @param streamProcessor The {@link ParallelStreamProcessor} to process records from + * @param pollCtx The context of the current consumer poll */ - void process(final ParallelStreamProcessor streamProcessor); + void handlePoll(final PollContext pollCtx); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java deleted file mode 100644 index 3ffd3e6f5..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessor.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.dependencytrack.event.kafka.processor.api; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; - -import java.util.List; - -/** - * A processor of individual {@link ConsumerRecord}s, capable of producing zero or more {@link ProducerRecord}s. - * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - * @param Type of the {@link ProducerRecord} key - * @param Type of the {@link ProducerRecord} value - */ -public interface RecordProcessor { - - /** - * Process a {@link ConsumerRecord}, and produce zero or more {@link ProducerRecord}s. - * - * @param record The {@link ConsumerRecord} to process - * @return Zero or more {@link ProducerRecord}s - * @throws RecordProcessingException When processing the {@link ConsumerRecord} failed - */ - List> process(final ConsumerRecord record) throws RecordProcessingException; - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java new file mode 100644 index 000000000..87b9fcaa2 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java @@ -0,0 +1,166 @@ +package org.dependencytrack.event.kafka.processor.api; + +import alpine.Config; +import alpine.common.logging.Logger; +import alpine.common.metrics.Metrics; +import io.confluent.parallelconsumer.ParallelConsumerOptions; +import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; +import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.dependencytrack.event.kafka.KafkaTopics.Topic; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.CommonClientConfigs.CLIENT_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; + +public class RecordProcessorManager implements AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(RecordProcessorManager.class); + + private final Map managedProcessors = new LinkedHashMap<>(); + private final Config config; + + public RecordProcessorManager() { + this(Config.getInstance()); + } + + RecordProcessorManager(final Config config) { + this.config = config; + } + + public void register(final String name, final SingleRecordProcessor recordProcessor, final Topic topic) { + final var processingStrategy = new SingleRecordProcessingStrategy<>(recordProcessor, topic.keySerde(), topic.valueSerde()); + final var parallelConsumer = createParallelConsumer(name, false); + managedProcessors.put(name, new ManagedRecordProcessor(parallelConsumer, processingStrategy, topic)); + } + + public void register(final String name, final BatchRecordProcessor recordProcessor, final Topic topic) { + final var processingStrategy = new BatchRecordProcessingStrategy<>(recordProcessor, topic.keySerde(), topic.valueSerde()); + final var parallelConsumer = createParallelConsumer(name, true); + managedProcessors.put(name, new ManagedRecordProcessor(parallelConsumer, processingStrategy, topic)); + } + + public void startAll() { + for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { + final String processorName = managedProcessorEntry.getKey(); + final ManagedRecordProcessor managedProcessor = managedProcessorEntry.getValue(); + + LOGGER.info("Starting processor %s".formatted(processorName)); + managedProcessor.parallelConsumer().subscribe(List.of(managedProcessor.topic().name())); + managedProcessor.parallelConsumer().poll(managedProcessor.processingStrategy::handlePoll); + } + } + + @Override + public void close() { + for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { + final String processorName = managedProcessorEntry.getKey(); + final ManagedRecordProcessor managedProcessor = managedProcessorEntry.getValue(); + + LOGGER.info("Stopping processor %s".formatted(processorName)); + managedProcessor.parallelConsumer().closeDrainFirst(); + } + } + + private ParallelStreamProcessor createParallelConsumer(final String processorName, final boolean isBatch) { + final var optionsBuilder = ParallelConsumerOptions.builder() + .consumer(createConsumer(processorName)); + + final Map properties = passThroughProperties(processorName); + + final ProcessingOrder processingOrder = Optional.ofNullable(properties.get("processing.order")) + .map(String::toUpperCase) + .map(ProcessingOrder::valueOf) + .orElse(ProcessingOrder.KEY); + optionsBuilder.ordering(processingOrder); + + final int maxConcurrency = Optional.ofNullable(properties.get("max.concurrency")) + .map(Integer::parseInt) + .orElse(3); + optionsBuilder.maxConcurrency(maxConcurrency); + + if (isBatch) { + final int maxBatchSize = Optional.ofNullable(properties.get("max.batch.size")) + .map(Integer::parseInt) + .orElse(10); + optionsBuilder.batchSize(maxBatchSize); + } + + if (Config.getInstance().getPropertyAsBoolean(Config.AlpineKey.METRICS_ENABLED)) { + optionsBuilder + .meterRegistry(Metrics.getRegistry()) + .pcInstanceTag(processorName); + } + + final ParallelConsumerOptions options = optionsBuilder.build(); + LOGGER.debug("Creating parallel consumer for processor %s with options %s".formatted(processorName, options)); + return ParallelStreamProcessor.createEosStreamProcessor(options); + } + + private Consumer createConsumer(final String processorName) { + final var consumerConfig = new HashMap(); + consumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); + consumerConfig.put(CLIENT_ID_CONFIG, "%s-consumer".formatted(processorName)); + consumerConfig.put(GROUP_ID_CONFIG, processorName); + consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); + + final Map properties = passThroughProperties("%s.consumer".formatted(processorName)); + for (final Map.Entry property : properties.entrySet()) { + if (!ConsumerConfig.configNames().contains(property.getKey())) { + LOGGER.warn("Consumer property %s was set for processor %s, but is unknown; Ignoring" + .formatted(property.getKey(), processorName)); + continue; + } + + consumerConfig.put(property.getKey(), property.getValue()); + } + + final var consumer = new KafkaConsumer(consumerConfig); + if (config.getPropertyAsBoolean(Config.AlpineKey.METRICS_ENABLED)) { + new KafkaClientMetrics(consumer).bindTo(Metrics.getRegistry()); + } + + return consumer; + } + + private Map passThroughProperties(final String prefix) { + final String fullPrefix = "kafka.processor.%s".formatted(prefix); + final Pattern fullPrefixPattern = Pattern.compile(Pattern.quote("%s.".formatted(fullPrefix))); + + final Map properties = config.getPassThroughProperties(fullPrefix); + if (properties.isEmpty()) { + return properties; + } + + final var trimmedProperties = new HashMap(properties.size()); + for (final Map.Entry property : properties.entrySet()) { + final String trimmedKey = fullPrefixPattern.matcher(property.getKey()).replaceFirst(""); + trimmedProperties.put(trimmedKey, property.getValue()); + } + + return trimmedProperties; + } + + private record ManagedRecordProcessor(ParallelStreamProcessor parallelConsumer, + RecordProcessingStrategy processingStrategy, + Topic topic) { + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java index 7e95796b9..dae53e84d 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java @@ -2,37 +2,28 @@ import alpine.common.logging.Logger; import io.confluent.parallelconsumer.PCRetriableException; -import io.confluent.parallelconsumer.ParallelStreamProcessor; import io.confluent.parallelconsumer.PollContext; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.Serde; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; -import java.util.Collections; -import java.util.List; - /** * A {@link RecordProcessingStrategy} that processes records individually. * - * @param Type of the {@link ConsumerRecord} key - * @param Type of the {@link ConsumerRecord} value - * @param Type of the {@link ProducerRecord} key - * @param Type of the {@link ProducerRecord} value + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value */ -public class SingleRecordProcessingStrategy extends AbstractRecordProcessingStrategy { +class SingleRecordProcessingStrategy extends AbstractRecordProcessingStrategy { private static final Logger LOGGER = Logger.getLogger(SingleRecordProcessingStrategy.class); - private final RecordProcessor recordProcessor; + private final SingleRecordProcessor recordProcessor; - public SingleRecordProcessingStrategy(final RecordProcessor recordProcessor, - final Deserializer keyDeserializer, final Deserializer valueDeserializer, - final Serializer keySerializer, final Serializer valueSerializer) { - super(keyDeserializer, valueDeserializer, keySerializer, valueSerializer); + SingleRecordProcessingStrategy(final SingleRecordProcessor recordProcessor, + final Serde keySerde, final Serde valueSerde) { + super(keySerde, valueSerde); this.recordProcessor = recordProcessor; } @@ -40,43 +31,28 @@ public SingleRecordProcessingStrategy(final RecordProcessor reco * {@inheritDoc} */ @Override - public void process(final ParallelStreamProcessor streamProcessor) { - if (canProduce(recordProcessor)) { - streamProcessor.pollAndProduceMany(this::handlePoll); - } else { - streamProcessor.poll(this::handlePoll); - } - } - - private List> handlePoll(final PollContext pollCtx) { + public void handlePoll(final PollContext pollCtx) { final ConsumerRecord record = pollCtx.getSingleConsumerRecord(); - final ConsumerRecord deserializedRecord; + final ConsumerRecord deserializedRecord; try { deserializedRecord = deserialize(record); } catch (SerializationException e) { LOGGER.error("Failed to deserialize consumer record %s; Skipping".formatted(record), e); // TODO: Consider supporting error handlers, e.g. to send record to DLT. - return Collections.emptyList(); // Skip record to avoid poison-pill scenario. + return; // Skip record to avoid poison-pill scenario. } - final List> producerRecords; try { - producerRecords = recordProcessor.process(deserializedRecord); + recordProcessor.process(deserializedRecord); } catch (RetryableRecordProcessingException e) { LOGGER.warn("Encountered retryable exception while processing %s".formatted(deserializedRecord), e); throw new PCRetriableException(e); } catch (RecordProcessingException | RuntimeException e) { LOGGER.error("Encountered non-retryable exception while processing %s; Skipping".formatted(deserializedRecord), e); // TODO: Consider supporting error handlers, e.g. to send record to DLT. - return Collections.emptyList(); // Skip record to avoid poison-pill scenario. + // Skip record to avoid poison-pill scenario. } - - return maybeSerializeAll(producerRecords); - } - - private static boolean canProduce(final RecordProcessor processor) { - return !RecordConsumer.class.isAssignableFrom(processor.getClass()); } } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java new file mode 100644 index 000000000..3ecf726b0 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java @@ -0,0 +1,22 @@ +package org.dependencytrack.event.kafka.processor.api; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; + +/** + * A processor of individual {@link ConsumerRecord}s. + * + * @param Type of the {@link ConsumerRecord} key + * @param Type of the {@link ConsumerRecord} value + */ +public interface SingleRecordProcessor { + + /** + * Process a {@link ConsumerRecord}. + * + * @param record The {@link ConsumerRecord} to process + * @throws RecordProcessingException When processing the {@link ConsumerRecord} failed + */ + void process(final ConsumerRecord record) throws RecordProcessingException; + +} diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java index e83f7d7f2..2a5deb598 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java @@ -4,15 +4,10 @@ import net.mguenther.kafka.junit.ExternalKafkaCluster; import net.mguenther.kafka.junit.KeyValue; import net.mguenther.kafka.junit.ReadKeyValues; -import net.mguenther.kafka.junit.SendKeyValues; import net.mguenther.kafka.junit.TopicConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.BooleanDeserializer; import org.apache.kafka.common.serialization.IntegerDeserializer; -import org.apache.kafka.common.serialization.LongSerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.dependencytrack.event.kafka.processor.api.RecordProcessor; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -22,8 +17,6 @@ import java.time.Duration; import java.util.List; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -45,17 +38,17 @@ public void testSingleRecordProcessor() throws Exception { kafka.createTopic(TopicConfig.withName("input")); kafka.createTopic(TopicConfig.withName("output")); - final RecordProcessor recordProcessor = - record -> List.of(new ProducerRecord<>("output", 2, true)); - - final var processorFactory = new KafkaProcessorFactory(); - try (final KafkaProcessor processor = processorFactory.createProcessor(recordProcessor)) { - processor.start(); - - kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", 123L))) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName())); - } +// final RecordProcessor recordProcessor = +// record -> List.of(new ProducerRecord<>("output", 2, true)); +// +// final var processorFactory = new KafkaRecordConsumerFactory(); +// try (final KafkaProcessor processor = processorFactory.createProcessor(recordProcessor)) { +// processor.start(); +// +// kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", 123L))) +// .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) +// .with(VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName())); +// } await() .atMost(Duration.ofSeconds(5)) diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java new file mode 100644 index 000000000..a775830c5 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java @@ -0,0 +1,112 @@ +package org.dependencytrack.event.kafka.processor.api; + +import alpine.Config; +import net.mguenther.kafka.junit.ExternalKafkaCluster; +import net.mguenther.kafka.junit.KeyValue; +import net.mguenther.kafka.junit.SendKeyValues; +import net.mguenther.kafka.junit.TopicConfig; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringSerializer; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.kafka.KafkaTopics.Topic; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.testcontainers.redpanda.RedpandaContainer; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +public class RecordProcessorManagerTest { + + @Rule + public RedpandaContainer container = new RedpandaContainer(DockerImageName + .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); + + ExternalKafkaCluster kafka; + Config configMock; + + @Before + public void setUp() { + kafka = ExternalKafkaCluster.at(container.getBootstrapServers()); + + configMock = mock(Config.class); + doReturn(container.getBootstrapServers()) + .when(configMock).getProperty(eq(ConfigKey.KAFKA_BOOTSTRAP_SERVERS)); + } + + @Test + public void test() throws Exception { + final var inputTopic = new Topic<>("input", Serdes.String(), Serdes.String()); + kafka.createTopic(TopicConfig.withName(inputTopic.name())); + + final var foo = new ConcurrentLinkedQueue(); + + final SingleRecordProcessor recordProcessor = + record -> foo.add("%s-%s".formatted(record.key(), record.value())); + + try (final var processorManager = new RecordProcessorManager(configMock)) { + processorManager.register("foo", recordProcessor, inputTopic); + processorManager.startAll(); + + kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", "bar"))) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName())); + + await() + .atMost(Duration.ofSeconds(5)) + .untilAsserted(() -> assertThat(foo).containsOnly("foo-bar")); + } + } + + @Test + public void testBatchProcessor() throws Exception { + final var inputTopic = new Topic<>("input", Serdes.String(), Serdes.String()); + kafka.createTopic(TopicConfig.withName(inputTopic.name()).withNumberOfPartitions(6)); + + final var recordsProcessed = new AtomicInteger(0); + final var actualBatchSizes = new ConcurrentLinkedQueue<>(); + + doReturn(Map.of( + "kafka.processor.foo.max.batch.size", "100" + )).when(configMock).getPassThroughProperties(eq("kafka.processor.foo")); + doReturn(Map.of("kafka.processor.foo.consumer.auto.offset.reset", "earliest")) + .when(configMock).getPassThroughProperties(eq("kafka.processor.foo.consumer")); + + final BatchRecordProcessor recordProcessor = records -> { + recordsProcessed.addAndGet(records.size()); + actualBatchSizes.add(records.size()); + }; + + try (final var processorManager = new RecordProcessorManager(configMock)) { + processorManager.register("foo", recordProcessor, inputTopic); + + for (int i = 0; i < 1_000; i++) { + kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo" + i, "bar" + i))) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName())); + } + + processorManager.startAll(); + + await() + .atMost(Duration.ofSeconds(5)) + .untilAsserted(() -> assertThat(recordsProcessed).hasValue(1_000)); + + assertThat(actualBatchSizes).containsOnly(100); + } + } + +} \ No newline at end of file From 275d832abc39cb0a6e722e9e2cec67f83689cfd4 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 16:38:50 +0100 Subject: [PATCH 05/26] WIP: Decouple processing strategy from parallel consumer specifics Signed-off-by: nscuro --- .../api/BatchRecordProcessingStrategy.java | 5 +--- .../api/RecordProcessingStrategy.java | 10 ++++--- .../processor/api/RecordProcessorManager.java | 15 +++++++++- .../api/SingleRecordProcessingStrategy.java | 14 ++++++++-- .../api/RecordProcessorManagerTest.java | 28 +++++++++++++------ 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java index 6034ae84c..f2da643b5 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java @@ -2,7 +2,6 @@ import alpine.common.logging.Logger; import io.confluent.parallelconsumer.PCRetriableException; -import io.confluent.parallelconsumer.PollContext; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; @@ -34,9 +33,7 @@ class BatchRecordProcessingStrategy extends AbstractRecordProcessingStrate * {@inheritDoc} */ @Override - public void handlePoll(final PollContext pollCtx) { - final List> records = pollCtx.getConsumerRecordsFlattened(); - + public void processRecords(final List> records) { final var deserializedRecords = new ArrayList>(records.size()); for (final ConsumerRecord record : records) { try { diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java index acd358601..475928bf5 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java @@ -1,14 +1,16 @@ package org.dependencytrack.event.kafka.processor.api; -import io.confluent.parallelconsumer.PollContext; +import org.apache.kafka.clients.consumer.ConsumerRecord; + +import java.util.List; interface RecordProcessingStrategy { /** - * Handle the result of a consumer poll. + * Process zero or more {@link ConsumerRecord}s. * - * @param pollCtx The context of the current consumer poll + * @param records The {@link ConsumerRecord}s to process */ - void handlePoll(final PollContext pollCtx); + void processRecords(final List> records); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java index 87b9fcaa2..99e0711d1 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java @@ -9,6 +9,7 @@ import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.dependencytrack.event.kafka.KafkaTopics.Topic; @@ -62,7 +63,10 @@ public void startAll() { LOGGER.info("Starting processor %s".formatted(processorName)); managedProcessor.parallelConsumer().subscribe(List.of(managedProcessor.topic().name())); - managedProcessor.parallelConsumer().poll(managedProcessor.processingStrategy::handlePoll); + managedProcessor.parallelConsumer().poll(pollCtx -> { + final List> polledRecords = pollCtx.getConsumerRecordsFlattened(); + managedProcessor.processingStrategy().processRecords(polledRecords); + }); } } @@ -95,6 +99,15 @@ private ParallelStreamProcessor createParallelConsumer(final Str optionsBuilder.maxConcurrency(maxConcurrency); if (isBatch) { + if (processingOrder == ProcessingOrder.PARTITION) { + LOGGER.warn(""" + Processor %s is configured to use batching with processing order %s; + Batch sizes are limited by the number of partitions in the topic, + and may not yield the desired effect \ + (https://github.com/confluentinc/parallel-consumer/issues/551)\ + """.formatted(processorName, processingOrder)); + } + final int maxBatchSize = Optional.ofNullable(properties.get("max.batch.size")) .map(Integer::parseInt) .orElse(10); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java index dae53e84d..a05a396b9 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java @@ -2,13 +2,14 @@ import alpine.common.logging.Logger; import io.confluent.parallelconsumer.PCRetriableException; -import io.confluent.parallelconsumer.PollContext; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; +import java.util.List; + /** * A {@link RecordProcessingStrategy} that processes records individually. * @@ -31,8 +32,15 @@ class SingleRecordProcessingStrategy extends AbstractRecordProcessingStrat * {@inheritDoc} */ @Override - public void handlePoll(final PollContext pollCtx) { - final ConsumerRecord record = pollCtx.getSingleConsumerRecord(); + public void processRecords(final List> records) { + if (records.isEmpty()) { + return; + } + if (records.size() > 1) { + throw new IllegalArgumentException("Expected at most one record, but received %d".formatted(records.size())); + } + + final ConsumerRecord record = records.get(0); final ConsumerRecord deserializedRecord; try { diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java index a775830c5..d48688b30 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java @@ -52,22 +52,31 @@ public void test() throws Exception { final var inputTopic = new Topic<>("input", Serdes.String(), Serdes.String()); kafka.createTopic(TopicConfig.withName(inputTopic.name())); - final var foo = new ConcurrentLinkedQueue(); + final var recordsProcessed = new AtomicInteger(0); + + doReturn(Map.of( + "kafka.processor.foo.processing.order", "key", + "kafka.processor.foo.max.concurrency", "5", + "kafka.processor.foo.consumer.auto.offset.reset", "earliest" + )).when(configMock).getPassThroughProperties(eq("kafka.processor.foo.consumer")); final SingleRecordProcessor recordProcessor = - record -> foo.add("%s-%s".formatted(record.key(), record.value())); + record -> recordsProcessed.incrementAndGet(); try (final var processorManager = new RecordProcessorManager(configMock)) { processorManager.register("foo", recordProcessor, inputTopic); - processorManager.startAll(); - kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", "bar"))) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName())); + for (int i = 0; i < 100; i++) { + kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo" + i, "bar" + i))) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName())); + } + + processorManager.startAll(); - await() + await("Record Processing") .atMost(Duration.ofSeconds(5)) - .untilAsserted(() -> assertThat(foo).containsOnly("foo-bar")); + .untilAsserted(() -> assertThat(recordsProcessed).hasValue(100)); } } @@ -80,6 +89,7 @@ public void testBatchProcessor() throws Exception { final var actualBatchSizes = new ConcurrentLinkedQueue<>(); doReturn(Map.of( + "kafka.processor.foo.processing.order", "key", "kafka.processor.foo.max.batch.size", "100" )).when(configMock).getPassThroughProperties(eq("kafka.processor.foo")); doReturn(Map.of("kafka.processor.foo.consumer.auto.offset.reset", "earliest")) @@ -101,7 +111,7 @@ public void testBatchProcessor() throws Exception { processorManager.startAll(); - await() + await("Record Processing") .atMost(Duration.ofSeconds(5)) .untilAsserted(() -> assertThat(recordsProcessed).hasValue(1_000)); From ff0cef81b724ef18cd49c45dd7dd89bec8afcedb Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 17:10:39 +0100 Subject: [PATCH 06/26] WIP: Handle retryable exceptions Signed-off-by: nscuro --- .../api/AbstractRecordProcessingStrategy.java | 25 +++++++++++++++++++ .../api/BatchRecordProcessingStrategy.java | 9 ++++--- .../api/SingleRecordProcessingStrategy.java | 9 ++++--- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java index da1cbfaec..5a9ab3b8c 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java @@ -1,8 +1,18 @@ package org.dependencytrack.event.kafka.processor.api; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.http.conn.ConnectTimeoutException; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; +import org.datanucleus.api.jdo.exceptions.ConnectionInUseException; +import org.datanucleus.store.query.QueryInterruptedException; +import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; + +import javax.jdo.JDOOptimisticVerificationException; +import java.net.SocketTimeoutException; +import java.sql.SQLTransientException; +import java.util.concurrent.TimeoutException; /** * An abstract {@link RecordProcessingStrategy} that provides various shared functionality. @@ -44,4 +54,19 @@ ConsumerRecord deserialize(final ConsumerRecord record) { deserializedKey, deserializedValue, record.headers(), record.leaderEpoch()); } + boolean isRetryableException(final Throwable throwable) { + if (throwable instanceof RetryableRecordProcessingException) { + return true; + } + + final Throwable rootCause = ExceptionUtils.getRootCause(throwable); + return rootCause instanceof TimeoutException + || rootCause instanceof ConnectTimeoutException + || rootCause instanceof SocketTimeoutException + || rootCause instanceof ConnectionInUseException + || rootCause instanceof QueryInterruptedException + || rootCause instanceof JDOOptimisticVerificationException + || rootCause instanceof SQLTransientException; + } + } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java index f2da643b5..bad98d165 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java @@ -6,7 +6,6 @@ import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; -import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; import java.util.ArrayList; import java.util.List; @@ -51,10 +50,12 @@ public void processRecords(final List> records) { try { batchConsumer.process(deserializedRecords); - } catch (RetryableRecordProcessingException e) { - LOGGER.warn("Encountered retryable exception while processing %d records".formatted(deserializedRecords.size()), e); - throw new PCRetriableException(e); } catch (RecordProcessingException | RuntimeException e) { + if (isRetryableException(e)) { + LOGGER.warn("Encountered retryable exception while processing %d records".formatted(deserializedRecords.size()), e); + throw new PCRetriableException(e); + } + LOGGER.error("Encountered non-retryable exception while processing %d records; Skipping".formatted(deserializedRecords.size()), e); // TODO: Consider supporting error handlers, e.g. to send records to DLT. // Skip records to avoid poison-pill scenario. diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java index a05a396b9..352e90c24 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java @@ -6,7 +6,6 @@ import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; -import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; import java.util.List; @@ -53,10 +52,12 @@ public void processRecords(final List> records) { try { recordProcessor.process(deserializedRecord); - } catch (RetryableRecordProcessingException e) { - LOGGER.warn("Encountered retryable exception while processing %s".formatted(deserializedRecord), e); - throw new PCRetriableException(e); } catch (RecordProcessingException | RuntimeException e) { + if (isRetryableException(e)) { + LOGGER.warn("Encountered retryable exception while processing %s".formatted(deserializedRecord), e); + throw new PCRetriableException(e); + } + LOGGER.error("Encountered non-retryable exception while processing %s; Skipping".formatted(deserializedRecord), e); // TODO: Consider supporting error handlers, e.g. to send record to DLT. // Skip record to avoid poison-pill scenario. From 34566eca69ab6a874f2bd53f19017a6f43b91296 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 17:11:02 +0100 Subject: [PATCH 07/26] WIP: Cleanup Signed-off-by: nscuro --- ...essedVulnerabilityScanResultProcessor.java | 17 ------------ .../VulnerabilityScanResultProcessor.java | 22 ---------------- .../streams/KafkaStreamsTopologyFactory.java | 26 +++++++------------ src/main/webapp/WEB-INF/web.xml | 3 +++ 4 files changed, 13 insertions(+), 55 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java deleted file mode 100644 index 3ca31c163..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.api.BatchRecordProcessor; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; - -import java.util.List; - -public class ProcessedVulnerabilityScanResultProcessor implements BatchRecordProcessor { - - @Override - public void process(final List> consumerRecords) throws RecordProcessingException { - // TODO: Group and aggregate by key (scan token) - // TODO: Use JDBI to batch upsert into "VULNERABILITYSCAN" table - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java deleted file mode 100644 index f25183bae..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; -import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; -import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; - -/** - * TODO: This should produce ProducerRecord, where ProcessedScanResult - * is a "summary" of the processed result that can be used to update the "VULNERABILITYSCAN" table. - * The String should be the scan token, so we can avoid concurrent writes to the same "VULNERABILITYSCAN" - * row when processing the respective records. - */ -public class VulnerabilityScanResultProcessor implements SingleRecordProcessor { - - @Override - public void process(final ConsumerRecord record) throws RecordProcessingException { - // TODO - } - -} diff --git a/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java b/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java index 12a5b6b15..026ff0686 100644 --- a/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java +++ b/src/main/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTopologyFactory.java @@ -9,14 +9,20 @@ import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.Topology; -import org.apache.kafka.streams.kstream.*; +import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.Repartitioned; import org.datanucleus.PropertyNames; import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.*; +import org.dependencytrack.event.ComponentMetricsUpdateEvent; +import org.dependencytrack.event.ComponentPolicyEvaluationEvent; +import org.dependencytrack.event.PortfolioVulnerabilityAnalysisEvent; +import org.dependencytrack.event.ProjectMetricsUpdateEvent; +import org.dependencytrack.event.ProjectPolicyEvaluationEvent; import org.dependencytrack.event.kafka.KafkaTopics; import org.dependencytrack.event.kafka.streams.processor.DelayedBomProcessedNotificationProcessor; -import org.dependencytrack.event.kafka.streams.processor.MirrorVulnerabilityProcessor; -import org.dependencytrack.event.kafka.streams.processor.RepositoryMetaResultProcessor; import org.dependencytrack.event.kafka.streams.processor.VulnerabilityScanResultProcessor; import org.dependencytrack.model.VulnerabilityScan; import org.dependencytrack.model.WorkflowState; @@ -210,18 +216,6 @@ Topology createTopology() { Event.dispatch(policyEvaluationEvent); }, Named.as("trigger_policy_evaluation")); - streamsBuilder - .stream(KafkaTopics.REPO_META_ANALYSIS_RESULT.name(), - Consumed.with(KafkaTopics.REPO_META_ANALYSIS_RESULT.keySerde(), KafkaTopics.REPO_META_ANALYSIS_RESULT.valueSerde()) - .withName("consume_from_%s_topic".formatted(KafkaTopics.REPO_META_ANALYSIS_RESULT.name()))) - .process(RepositoryMetaResultProcessor::new, Named.as("process_repo_meta_analysis_result")); - - streamsBuilder - .stream(KafkaTopics.NEW_VULNERABILITY.name(), - Consumed.with(KafkaTopics.NEW_VULNERABILITY.keySerde(), KafkaTopics.NEW_VULNERABILITY.valueSerde()) - .withName("consume_from_%s_topic".formatted(KafkaTopics.NEW_VULNERABILITY.name()))) - .process(MirrorVulnerabilityProcessor::new, Named.as("process_mirror_vulnerability")); - return streamsBuilder.build(streamsProperties); } diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index f7f1c5d63..d6b206117 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -47,6 +47,9 @@ org.dependencytrack.event.EventSubsystemInitializer + + org.dependencytrack.event.kafka.processor.KafkaRecordProcessorInitializer + org.dependencytrack.event.kafka.streams.KafkaStreamsInitializer From 215a51cb98e716b101ff0b0f1a3f25824e2722b3 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 17:11:17 +0100 Subject: [PATCH 08/26] WIP: Add processor application properties Signed-off-by: nscuro --- src/main/resources/application.properties | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index e3db0238c..25f17b0d6 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -434,6 +434,11 @@ kafka.streams.production.exception.threshold.interval=PT30M kafka.streams.transient.processing.exception.threshold.count=50 kafka.streams.transient.processing.exception.threshold.interval=PT30M +alpine.kafka.processor.repo-meta-result.max.concurrency=3 +alpine.kafka.processor.repo-meta-result.processing.order=key +alpine.kafka.processor.vuln-mirror.max.concurrency=3 +alpine.kafka.processor.vuln-mirror.processing.order=key + # Scheduling tasks after 3 minutes (3*60*1000) of starting application task.scheduler.initial.delay=180000 From 5bd649dd066ca75af934ccab30b44ae7de9809e5 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 17:19:06 +0100 Subject: [PATCH 09/26] WIP: Add more processor application properties Signed-off-by: nscuro --- .../kafka/processor/KafkaRecordProcessorInitializer.java | 6 ++++-- .../kafka/processor/RepositoryMetaResultProcessor.java | 2 ++ .../kafka/processor/VulnerabilityMirrorProcessor.java | 2 ++ src/main/resources/application.properties | 7 +++++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java index ebcfaf229..e93314008 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java @@ -17,8 +17,10 @@ public class KafkaRecordProcessorInitializer implements ServletContextListener { public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Initializing Kafka processors"); - processorManager.register("vuln-mirror", new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); - processorManager.register("repo-meta-result", new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); + processorManager.register(VulnerabilityMirrorProcessor.PROCESSOR_NAME, + new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); + processorManager.register(RepositoryMetaResultProcessor.PROCESSOR_NAME, + new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); processorManager.startAll(); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index 64909b0c3..1e7ca5e83 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -32,6 +32,8 @@ */ public class RepositoryMetaResultProcessor implements SingleRecordProcessor { + public static final String PROCESSOR_NAME = "repo-meta-result"; + private static final Logger LOGGER = Logger.getLogger(org.dependencytrack.event.kafka.streams.processor.RepositoryMetaResultProcessor.class); private static final Timer TIMER = Timer.builder("repo_meta_result_processing") .description("Time taken to process repository meta analysis results") diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java index d6dbabc9c..ee15df48a 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java @@ -34,6 +34,8 @@ */ public class VulnerabilityMirrorProcessor implements SingleRecordProcessor { + public static final String PROCESSOR_NAME = "vuln-mirror"; + private static final Logger LOGGER = Logger.getLogger(MirrorVulnerabilityProcessor.class); private static final Timer TIMER = Timer.builder("vuln_mirror_processing") .description("Time taken to process mirrored vulnerabilities") diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 25f17b0d6..8310b4eaf 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -434,10 +434,17 @@ kafka.streams.production.exception.threshold.interval=PT30M kafka.streams.transient.processing.exception.threshold.count=50 kafka.streams.transient.processing.exception.threshold.interval=PT30M +# Optional +# alpine.kafka.processor..processing.order= +# alpine.kafka.processor..max.batch.size= +# alpine.kafka.processor..max.concurrency= +# alpine.kafka.processor..consumer.= alpine.kafka.processor.repo-meta-result.max.concurrency=3 alpine.kafka.processor.repo-meta-result.processing.order=key +alpine.kafka.processor.repo-meta-result.consumer.group.id=dtrack-apiserver-processor alpine.kafka.processor.vuln-mirror.max.concurrency=3 alpine.kafka.processor.vuln-mirror.processing.order=key +alpine.kafka.processor.vuln-mirror.consumer.group.id=dtrack-apiserver-processor # Scheduling tasks after 3 minutes (3*60*1000) of starting application task.scheduler.initial.delay=180000 From f2b205a8cd80f8c32d0f6aa6526ba90dd724480b Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 19:34:25 +0100 Subject: [PATCH 10/26] WIP: Add customizable consumer retry delays Signed-off-by: nscuro --- .../processor/api/RecordProcessorManager.java | 60 +++++++++++++++---- src/main/resources/application.properties | 12 +++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java index 99e0711d1..9c4b56cfe 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java @@ -6,6 +6,7 @@ import io.confluent.parallelconsumer.ParallelConsumerOptions; import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; import io.confluent.parallelconsumer.ParallelStreamProcessor; +import io.github.resilience4j.core.IntervalFunction; import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -14,6 +15,7 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.dependencytrack.event.kafka.KafkaTopics.Topic; +import java.time.Duration; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -33,6 +35,21 @@ public class RecordProcessorManager implements AutoCloseable { private static final Logger LOGGER = Logger.getLogger(RecordProcessorManager.class); + private static final String PROPERTY_MAX_BATCH_SIZE = "max.batch.size"; + private static final int PROPERTY_MAX_BATCH_SIZE_DEFAULT = 10; + private static final String PROPERTY_MAX_CONCURRENCY = "max.concurrency"; + private static final int PROPERTY_MAX_CONCURRENCY_DEFAULT = 3; + private static final String PROPERTY_PROCESSING_ORDER = "processing.order"; + private static final ProcessingOrder PROPERTY_PROCESSING_ORDER_DEFAULT = ProcessingOrder.KEY; + private static final String PROPERTY_RETRY_INITIAL_DELAY = "retry.initial.delay"; + private static final Duration PROPERTY_RETRY_INITIAL_DELAY_DEFAULT = Duration.ofSeconds(1); + private static final String PROPERTY_RETRY_MULTIPLIER = "retry.multiplier"; + private static final int PROPERTY_RETRY_MULTIPLIER_DEFAULT = 1; + private static final String PROPERTY_RETRY_RANDOMIZATION_FACTOR = "retry.randomization.factor"; + private static final double PROPERTY_RETRY_RANDOMIZATION_FACTOR_DEFAULT = 0.3; + private static final String PROPERTY_RETRY_MAX_DELAY = "retry.max.delay"; + private static final Duration PROPERTY_RETRY_MAX_DELAY_DEFAULT = Duration.ofMinutes(1); + private final Map managedProcessors = new LinkedHashMap<>(); private final Config config; @@ -85,17 +102,17 @@ private ParallelStreamProcessor createParallelConsumer(final Str final var optionsBuilder = ParallelConsumerOptions.builder() .consumer(createConsumer(processorName)); - final Map properties = passThroughProperties(processorName); + final Map properties = getPassThroughProperties(processorName); - final ProcessingOrder processingOrder = Optional.ofNullable(properties.get("processing.order")) + final ProcessingOrder processingOrder = Optional.ofNullable(properties.get(PROPERTY_PROCESSING_ORDER)) .map(String::toUpperCase) .map(ProcessingOrder::valueOf) - .orElse(ProcessingOrder.KEY); + .orElse(PROPERTY_PROCESSING_ORDER_DEFAULT); optionsBuilder.ordering(processingOrder); - final int maxConcurrency = Optional.ofNullable(properties.get("max.concurrency")) + final int maxConcurrency = Optional.ofNullable(properties.get(PROPERTY_MAX_CONCURRENCY)) .map(Integer::parseInt) - .orElse(3); + .orElse(PROPERTY_MAX_CONCURRENCY_DEFAULT); optionsBuilder.maxConcurrency(maxConcurrency); if (isBatch) { @@ -103,17 +120,23 @@ private ParallelStreamProcessor createParallelConsumer(final Str LOGGER.warn(""" Processor %s is configured to use batching with processing order %s; Batch sizes are limited by the number of partitions in the topic, - and may not yield the desired effect \ + and may thus not yield the desired effect \ (https://github.com/confluentinc/parallel-consumer/issues/551)\ """.formatted(processorName, processingOrder)); } - final int maxBatchSize = Optional.ofNullable(properties.get("max.batch.size")) + final int maxBatchSize = Optional.ofNullable(properties.get(PROPERTY_MAX_BATCH_SIZE)) .map(Integer::parseInt) - .orElse(10); + .orElse(PROPERTY_MAX_BATCH_SIZE_DEFAULT); optionsBuilder.batchSize(maxBatchSize); } + final IntervalFunction retryIntervalFunction = getRetryIntervalFunction(properties); + optionsBuilder.retryDelayProvider(recordCtx -> { + final long delayMillis = retryIntervalFunction.apply(recordCtx.getNumberOfFailedAttempts()); + return Duration.ofMillis(delayMillis); + }); + if (Config.getInstance().getPropertyAsBoolean(Config.AlpineKey.METRICS_ENABLED)) { optionsBuilder .meterRegistry(Metrics.getRegistry()) @@ -134,7 +157,7 @@ private Consumer createConsumer(final String processorName) { consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); - final Map properties = passThroughProperties("%s.consumer".formatted(processorName)); + final Map properties = getPassThroughProperties("%s.consumer".formatted(processorName)); for (final Map.Entry property : properties.entrySet()) { if (!ConsumerConfig.configNames().contains(property.getKey())) { LOGGER.warn("Consumer property %s was set for processor %s, but is unknown; Ignoring" @@ -153,7 +176,7 @@ private Consumer createConsumer(final String processorName) { return consumer; } - private Map passThroughProperties(final String prefix) { + private Map getPassThroughProperties(final String prefix) { final String fullPrefix = "kafka.processor.%s".formatted(prefix); final Pattern fullPrefixPattern = Pattern.compile(Pattern.quote("%s.".formatted(fullPrefix))); @@ -171,6 +194,23 @@ private Map passThroughProperties(final String prefix) { return trimmedProperties; } + private static IntervalFunction getRetryIntervalFunction(final Map properties) { + final Duration initialDelay = Optional.ofNullable(properties.get(PROPERTY_RETRY_INITIAL_DELAY)) + .map(Duration::parse) + .orElse(PROPERTY_RETRY_INITIAL_DELAY_DEFAULT); + final Duration maxDelay = Optional.ofNullable(properties.get(PROPERTY_RETRY_MAX_DELAY)) + .map(Duration::parse) + .orElse(PROPERTY_RETRY_MAX_DELAY_DEFAULT); + final int multiplier = Optional.of(properties.get(PROPERTY_RETRY_MULTIPLIER)) + .map(Integer::parseInt) + .orElse(PROPERTY_RETRY_MULTIPLIER_DEFAULT); + final double randomizationFactor = Optional.of(properties.get(PROPERTY_RETRY_RANDOMIZATION_FACTOR)) + .map(Double::parseDouble) + .orElse(PROPERTY_RETRY_RANDOMIZATION_FACTOR_DEFAULT); + + return IntervalFunction.ofExponentialRandomBackoff(initialDelay, multiplier, randomizationFactor, maxDelay); + } + private record ManagedRecordProcessor(ParallelStreamProcessor parallelConsumer, RecordProcessingStrategy processingStrategy, Topic topic) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 8310b4eaf..73ada58bc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -435,15 +435,23 @@ kafka.streams.transient.processing.exception.threshold.count=50 kafka.streams.transient.processing.exception.threshold.interval=PT30M # Optional -# alpine.kafka.processor..processing.order= +# alpine.kafka.processor..processing.order=key # alpine.kafka.processor..max.batch.size= -# alpine.kafka.processor..max.concurrency= +# alpine.kafka.processor..max.concurrency=1 +# alpine.kafka.processor..retry.initial.delay=PT1S +# alpine.kafka.processor..retry.multiplier=1 +# alpine.kafka.processor..retry.randomization.factor=0.3 +# alpine.kafka.processor..retry.max.delay=PT1M # alpine.kafka.processor..consumer.= alpine.kafka.processor.repo-meta-result.max.concurrency=3 alpine.kafka.processor.repo-meta-result.processing.order=key alpine.kafka.processor.repo-meta-result.consumer.group.id=dtrack-apiserver-processor alpine.kafka.processor.vuln-mirror.max.concurrency=3 alpine.kafka.processor.vuln-mirror.processing.order=key +alpine.kafka.processor.vuln-mirror.retry.initial.delay=PT3S +alpine.kafka.processor.vuln-mirror.retry.multiplier=2 +alpine.kafka.processor.vuln-mirror.retry.randomization.factor=0.3 +alpine.kafka.processor.vuln-mirror.retry.max.delay=PT3M alpine.kafka.processor.vuln-mirror.consumer.group.id=dtrack-apiserver-processor # Scheduling tasks after 3 minutes (3*60*1000) of starting application From 8cc614fde9415ecce33b6187c5f2f1aebae55591 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 19:57:09 +0100 Subject: [PATCH 11/26] WIP: Specify retry delays in milliseconds `Duration` only works on second level Signed-off-by: nscuro --- .../processor/api/RecordProcessorManager.java | 27 +++--- src/main/resources/application.properties | 8 +- .../api/RecordProcessorManagerTest.java | 87 +++++++++++++++---- 3 files changed, 87 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java index 9c4b56cfe..8c6884708 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java @@ -41,14 +41,14 @@ public class RecordProcessorManager implements AutoCloseable { private static final int PROPERTY_MAX_CONCURRENCY_DEFAULT = 3; private static final String PROPERTY_PROCESSING_ORDER = "processing.order"; private static final ProcessingOrder PROPERTY_PROCESSING_ORDER_DEFAULT = ProcessingOrder.KEY; - private static final String PROPERTY_RETRY_INITIAL_DELAY = "retry.initial.delay"; - private static final Duration PROPERTY_RETRY_INITIAL_DELAY_DEFAULT = Duration.ofSeconds(1); + private static final String PROPERTY_RETRY_INITIAL_DELAY_MS = "retry.initial.delay.ms"; + private static final long PROPERTY_RETRY_INITIAL_DELAY_MS_DEFAULT = 1000; private static final String PROPERTY_RETRY_MULTIPLIER = "retry.multiplier"; private static final int PROPERTY_RETRY_MULTIPLIER_DEFAULT = 1; private static final String PROPERTY_RETRY_RANDOMIZATION_FACTOR = "retry.randomization.factor"; private static final double PROPERTY_RETRY_RANDOMIZATION_FACTOR_DEFAULT = 0.3; - private static final String PROPERTY_RETRY_MAX_DELAY = "retry.max.delay"; - private static final Duration PROPERTY_RETRY_MAX_DELAY_DEFAULT = Duration.ofMinutes(1); + private static final String PROPERTY_RETRY_MAX_DELAY_MS = "retry.max.delay.ms"; + private static final long PROPERTY_RETRY_MAX_DELAY_MS_DEFAULT = 60 * 1000; private final Map managedProcessors = new LinkedHashMap<>(); private final Config config; @@ -195,20 +195,21 @@ private Map getPassThroughProperties(final String prefix) { } private static IntervalFunction getRetryIntervalFunction(final Map properties) { - final Duration initialDelay = Optional.ofNullable(properties.get(PROPERTY_RETRY_INITIAL_DELAY)) - .map(Duration::parse) - .orElse(PROPERTY_RETRY_INITIAL_DELAY_DEFAULT); - final Duration maxDelay = Optional.ofNullable(properties.get(PROPERTY_RETRY_MAX_DELAY)) - .map(Duration::parse) - .orElse(PROPERTY_RETRY_MAX_DELAY_DEFAULT); - final int multiplier = Optional.of(properties.get(PROPERTY_RETRY_MULTIPLIER)) + final long initialDelayMs = Optional.ofNullable(properties.get(PROPERTY_RETRY_INITIAL_DELAY_MS)) + .map(Long::parseLong) + .orElse(PROPERTY_RETRY_INITIAL_DELAY_MS_DEFAULT); + final long maxDelayMs = Optional.ofNullable(properties.get(PROPERTY_RETRY_MAX_DELAY_MS)) + .map(Long::parseLong) + .orElse(PROPERTY_RETRY_MAX_DELAY_MS_DEFAULT); + final int multiplier = Optional.ofNullable(properties.get(PROPERTY_RETRY_MULTIPLIER)) .map(Integer::parseInt) .orElse(PROPERTY_RETRY_MULTIPLIER_DEFAULT); - final double randomizationFactor = Optional.of(properties.get(PROPERTY_RETRY_RANDOMIZATION_FACTOR)) + final double randomizationFactor = Optional.ofNullable(properties.get(PROPERTY_RETRY_RANDOMIZATION_FACTOR)) .map(Double::parseDouble) .orElse(PROPERTY_RETRY_RANDOMIZATION_FACTOR_DEFAULT); - return IntervalFunction.ofExponentialRandomBackoff(initialDelay, multiplier, randomizationFactor, maxDelay); + return IntervalFunction.ofExponentialRandomBackoff(Duration.ofMillis(initialDelayMs), + multiplier, randomizationFactor, Duration.ofMillis(maxDelayMs)); } private record ManagedRecordProcessor(ParallelStreamProcessor parallelConsumer, diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 73ada58bc..0dd346c5f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -438,20 +438,20 @@ kafka.streams.transient.processing.exception.threshold.interval=PT30M # alpine.kafka.processor..processing.order=key # alpine.kafka.processor..max.batch.size= # alpine.kafka.processor..max.concurrency=1 -# alpine.kafka.processor..retry.initial.delay=PT1S +# alpine.kafka.processor..retry.initial.delay.ms=1000 # alpine.kafka.processor..retry.multiplier=1 # alpine.kafka.processor..retry.randomization.factor=0.3 -# alpine.kafka.processor..retry.max.delay=PT1M +# alpine.kafka.processor..retry.max.delay.ms=60000 # alpine.kafka.processor..consumer.= alpine.kafka.processor.repo-meta-result.max.concurrency=3 alpine.kafka.processor.repo-meta-result.processing.order=key alpine.kafka.processor.repo-meta-result.consumer.group.id=dtrack-apiserver-processor alpine.kafka.processor.vuln-mirror.max.concurrency=3 alpine.kafka.processor.vuln-mirror.processing.order=key -alpine.kafka.processor.vuln-mirror.retry.initial.delay=PT3S +alpine.kafka.processor.vuln-mirror.retry.initial.delay.ms=3000 alpine.kafka.processor.vuln-mirror.retry.multiplier=2 alpine.kafka.processor.vuln-mirror.retry.randomization.factor=0.3 -alpine.kafka.processor.vuln-mirror.retry.max.delay=PT3M +alpine.kafka.processor.vuln-mirror.retry.max.delay.ms=180000 alpine.kafka.processor.vuln-mirror.consumer.group.id=dtrack-apiserver-processor # Scheduling tasks after 3 minutes (3*60*1000) of starting application diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java index d48688b30..ea856ba65 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; @@ -26,13 +27,14 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class RecordProcessorManagerTest { @Rule - public RedpandaContainer container = new RedpandaContainer(DockerImageName + public RedpandaContainer kafkaContainer = new RedpandaContainer(DockerImageName .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); ExternalKafkaCluster kafka; @@ -40,25 +42,26 @@ public class RecordProcessorManagerTest { @Before public void setUp() { - kafka = ExternalKafkaCluster.at(container.getBootstrapServers()); + kafka = ExternalKafkaCluster.at(kafkaContainer.getBootstrapServers()); configMock = mock(Config.class); - doReturn(container.getBootstrapServers()) - .when(configMock).getProperty(eq(ConfigKey.KAFKA_BOOTSTRAP_SERVERS)); + when(configMock.getProperty(eq(ConfigKey.KAFKA_BOOTSTRAP_SERVERS))) + .thenReturn(kafkaContainer.getBootstrapServers()); } @Test public void test() throws Exception { final var inputTopic = new Topic<>("input", Serdes.String(), Serdes.String()); - kafka.createTopic(TopicConfig.withName(inputTopic.name())); + kafka.createTopic(TopicConfig.withName(inputTopic.name()).withNumberOfPartitions(3)); final var recordsProcessed = new AtomicInteger(0); - doReturn(Map.of( - "kafka.processor.foo.processing.order", "key", - "kafka.processor.foo.max.concurrency", "5", - "kafka.processor.foo.consumer.auto.offset.reset", "earliest" - )).when(configMock).getPassThroughProperties(eq("kafka.processor.foo.consumer")); + when(configMock.getPassThroughProperties(eq("kafka.processor.foo.consumer"))) + .thenReturn(Map.of( + "kafka.processor.foo.processing.order", "key", + "kafka.processor.foo.max.concurrency", "5", + "kafka.processor.foo.consumer.auto.offset.reset", "earliest" + )); final SingleRecordProcessor recordProcessor = record -> recordsProcessed.incrementAndGet(); @@ -80,20 +83,68 @@ record -> recordsProcessed.incrementAndGet(); } } + @Test + public void testSingleRecordProcessorRetry() throws Exception { + final var inputTopic = new Topic<>("input", Serdes.String(), Serdes.String()); + kafka.createTopic(TopicConfig.withName(inputTopic.name()).withNumberOfPartitions(3)); + + final var attemptsCounter = new AtomicInteger(0); + + final var objectSpy = spy(new Object()); + when(objectSpy.toString()) + .thenThrow(new RuntimeException(new TimeoutException())) + .thenThrow(new RuntimeException(new TimeoutException())) + .thenThrow(new RuntimeException(new TimeoutException())) + .thenReturn("done"); + + final SingleRecordProcessor recordProcessor = record -> { + attemptsCounter.incrementAndGet(); + objectSpy.toString(); + }; + + when(configMock.getPassThroughProperties(eq("kafka.processor.foo"))) + .thenReturn(Map.of( + "kafka.processor.foo.retry.initial.delay.ms", "5", + "kafka.processor.foo.retry.multiplier", "1", + "kafka.processor.foo.retry.max.delay.ms", "10" + )); + when(configMock.getPassThroughProperties(eq("kafka.processor.foo.consumer"))) + .thenReturn(Map.of( + "kafka.processor.foo.consumer.auto.offset.reset", "earliest" + )); + + try (final var processorManager = new RecordProcessorManager(configMock)) { + processorManager.register("foo", recordProcessor, inputTopic); + + kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", "bar"))) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName())); + + processorManager.startAll(); + + await("Record Processing") + .atMost(Duration.ofSeconds(5)) + .untilAsserted(() -> assertThat(attemptsCounter).hasValue(4)); + } + } + @Test public void testBatchProcessor() throws Exception { final var inputTopic = new Topic<>("input", Serdes.String(), Serdes.String()); - kafka.createTopic(TopicConfig.withName(inputTopic.name()).withNumberOfPartitions(6)); + kafka.createTopic(TopicConfig.withName(inputTopic.name()).withNumberOfPartitions(3)); final var recordsProcessed = new AtomicInteger(0); final var actualBatchSizes = new ConcurrentLinkedQueue<>(); - doReturn(Map.of( - "kafka.processor.foo.processing.order", "key", - "kafka.processor.foo.max.batch.size", "100" - )).when(configMock).getPassThroughProperties(eq("kafka.processor.foo")); - doReturn(Map.of("kafka.processor.foo.consumer.auto.offset.reset", "earliest")) - .when(configMock).getPassThroughProperties(eq("kafka.processor.foo.consumer")); + when(configMock.getPassThroughProperties(eq("kafka.processor.foo"))) + .thenReturn(Map.of( + "kafka.processor.foo.processing.order", "key", + "kafka.processor.foo.max.batch.size", "100" + )); + when(configMock.getPassThroughProperties(eq("kafka.processor.foo.consumer"))) + .thenReturn(Map.of( + "kafka.processor.foo.consumer.auto.offset.reset", "earliest" + )); final BatchRecordProcessor recordProcessor = records -> { recordsProcessed.addAndGet(records.size()); From c1e488938d5bcba03c5a4871be66df6168ecbe6a Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 21:00:01 +0100 Subject: [PATCH 12/26] WIP: Migrate `MirrorVulnerabilityProcessor` Signed-off-by: nscuro --- .../VulnerabilityMirrorProcessor.java | 3 +- .../processor/api/RecordProcessorManager.java | 17 +- .../MirrorVulnerabilityProcessor.java | 307 ------------------ src/main/resources/application.properties | 6 +- .../kafka/processor/KafkaProcessorTest.java | 64 ---- .../VulnerabilityMirrorProcessorTest.java} | 169 +++++++--- .../api/RecordProcessorManagerTest.java | 4 +- .../event/kafka/streams/KafkaStreamsTest.java | 76 ----- 8 files changed, 143 insertions(+), 503 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java delete mode 100644 src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java rename src/test/java/org/dependencytrack/event/kafka/{streams/processor/MirrorVulnerabilityProcessorTest.java => processor/VulnerabilityMirrorProcessorTest.java} (88%) delete mode 100644 src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java index ee15df48a..786300728 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java @@ -12,7 +12,6 @@ import org.cyclonedx.proto.v1_4.VulnerabilityAffects; import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; -import org.dependencytrack.event.kafka.streams.processor.MirrorVulnerabilityProcessor; import org.dependencytrack.model.Vulnerability; import org.dependencytrack.model.VulnerableSoftware; import org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln; @@ -36,7 +35,7 @@ public class VulnerabilityMirrorProcessor implements SingleRecordProcessor createParallelConsumer(final Str .map(String::toUpperCase) .map(ProcessingOrder::valueOf) .orElse(PROPERTY_PROCESSING_ORDER_DEFAULT); + if (processingOrder != ProcessingOrder.PARTITION) { + LOGGER.warn(""" + Processor %s is configured to use ordering mode %s; \ + Ordering modes other than %s bypass head-of-line blocking \ + and can put additional strain on the system in cases where \ + external systems like the database are temporarily unavailable \ + (https://github.com/confluentinc/parallel-consumer#head-of-line-blocking)\ + """.formatted(processorName, processingOrder, ProcessingOrder.PARTITION)); + } optionsBuilder.ordering(processingOrder); final int maxConcurrency = Optional.ofNullable(properties.get(PROPERTY_MAX_CONCURRENCY)) @@ -118,8 +127,8 @@ private ParallelStreamProcessor createParallelConsumer(final Str if (isBatch) { if (processingOrder == ProcessingOrder.PARTITION) { LOGGER.warn(""" - Processor %s is configured to use batching with processing order %s; - Batch sizes are limited by the number of partitions in the topic, + Processor %s is configured to use batching with processing order %s; \ + Batch sizes are limited by the number of partitions in the topic, \ and may thus not yield the desired effect \ (https://github.com/confluentinc/parallel-consumer/issues/551)\ """.formatted(processorName, processingOrder)); diff --git a/src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java b/src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java deleted file mode 100644 index ed80d4d4d..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessor.java +++ /dev/null @@ -1,307 +0,0 @@ -package org.dependencytrack.event.kafka.streams.processor; - -import alpine.common.logging.Logger; -import alpine.common.metrics.Metrics; -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import io.micrometer.core.instrument.Timer; -import org.apache.commons.lang3.StringUtils; -import org.apache.kafka.streams.processor.api.Processor; -import org.apache.kafka.streams.processor.api.Record; -import org.cyclonedx.proto.v1_4.Bom; -import org.cyclonedx.proto.v1_4.Component; -import org.cyclonedx.proto.v1_4.VulnerabilityAffects; -import org.dependencytrack.model.Vulnerability; -import org.dependencytrack.model.VulnerableSoftware; -import org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln; -import org.dependencytrack.parser.nvd.ModelConverter; -import org.dependencytrack.parser.vers.Comparator; -import org.dependencytrack.parser.vers.Constraint; -import org.dependencytrack.parser.vers.Vers; -import org.dependencytrack.parser.vers.VersException; -import org.dependencytrack.persistence.QueryManager; -import us.springett.parsers.cpe.exceptions.CpeEncodingException; -import us.springett.parsers.cpe.exceptions.CpeParsingException; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - - -public class MirrorVulnerabilityProcessor implements Processor { - - private static final Logger LOGGER = Logger.getLogger(MirrorVulnerabilityProcessor.class); - private static final Timer TIMER = Timer.builder("vuln_mirror_processing") - .description("Time taken to process mirrored vulnerabilities") - .register(Metrics.getRegistry()); - - @Override - public void process(final Record record) { - final Timer.Sample timerSample = Timer.start(); - - try (QueryManager qm = new QueryManager().withL2CacheDisabled()) { - LOGGER.debug("Synchronizing Mirrored Vulnerability : " + record.key()); - Bom bom = record.value(); - String key = record.key(); - String mirrorSource = key.substring(0, key.indexOf("/")); - Vulnerability.Source source = Vulnerability.Source.valueOf(mirrorSource); - final Vulnerability vulnerability = ModelConverterCdxToVuln.convert(qm, bom, bom.getVulnerabilities(0), false); - final List vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(vulnerability.getSource(), vulnerability.getVulnId())); - final Vulnerability synchronizedVulnerability = qm.synchronizeVulnerability(vulnerability, false); - var cycloneVuln = bom.getVulnerabilities(0); - // Alias synchronization across multiple sources is too unreliable right now. - // We can re-enable this once we have more confidence in data quality, or a better - // way of auditing reported aliases. See also: https://github.com/google/osv.dev/issues/888 -// if (!cycloneVuln.getReferencesList().isEmpty()) { -// cycloneVuln.getReferencesList().stream().forEach(reference -> { -// final String alias = reference.getId(); -// final VulnerabilityAlias vulnerabilityAlias = new VulnerabilityAlias(); -// -// // OSV will use IDs of other vulnerability databases for its -// // primary advisory ID (e.g. GHSA-45hx-wfhj-473x). We need to ensure -// // that we don't falsely report GHSA IDs as stemming from OSV. -// final Vulnerability.Source advisorySource = extractSource(cycloneVuln.getId(), cycloneVuln.getSource()); -// if (mirrorSource.equals("OSV")) { -// switch (advisorySource) { -// case NVD -> vulnerabilityAlias.setCveId(cycloneVuln.getId()); -// case GITHUB -> vulnerabilityAlias.setGhsaId(cycloneVuln.getId()); -// default -> vulnerabilityAlias.setOsvId(cycloneVuln.getId()); -// } -// } -// if (alias.startsWith("CVE") && Vulnerability.Source.NVD != advisorySource) { -// vulnerabilityAlias.setCveId(alias); -// qm.synchronizeVulnerabilityAlias(vulnerabilityAlias); -// } else if (alias.startsWith("GHSA") && Vulnerability.Source.GITHUB != advisorySource) { -// vulnerabilityAlias.setGhsaId(alias); -// qm.synchronizeVulnerabilityAlias(vulnerabilityAlias); -// } -// }); -// } - final List vsList = new ArrayList<>(); - for (final VulnerabilityAffects affect : cycloneVuln.getAffectsList()) { - final Optional component = bom.getComponentsList().stream() - .filter(c -> c.getBomRef().equals(affect.getRef())) - .findFirst(); - if (component.isEmpty()) { - LOGGER.warn("No component in the BOV for %s is matching the BOM ref \"%s\" of the affects node; Skipping" - .formatted(synchronizedVulnerability.getVulnId(), affect.getRef())); - continue; - } - - affect.getVersionsList().forEach(version -> { - if (version.hasRange()) { - final VulnerableSoftware vs = mapAffectedRangeToVulnerableSoftware(qm, - vulnerability.getVulnId(), version.getRange(), component.get().getPurl(), component.get().getCpe()); - if (vs != null) { - vsList.add(vs); - } - } - if (version.hasVersion()) { - final VulnerableSoftware vs = mapAffectedVersionToVulnerableSoftware(qm, - vulnerability.getVulnId(), version.getVersion(), component.get().getPurl(), component.get().getCpe()); - if (vs != null) { - vsList.add(vs); - } - } - }); - } - if (!vsList.isEmpty()) { - qm.persist(vsList); - qm.updateAffectedVersionAttributions(synchronizedVulnerability, vsList, source); - var reconciledVsList = qm.reconcileVulnerableSoftware(synchronizedVulnerability, vsListOld, vsList, source); - synchronizedVulnerability.setVulnerableSoftware(reconciledVsList); - } - qm.persist(synchronizedVulnerability); - // Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, Vulnerability.class)); - } catch (Exception e) { - // TODO: Send record to a dead letter topic. - LOGGER.error("Synchronizing vulnerability %s failed".formatted(record.key()), e); - } finally { - timerSample.stop(TIMER); - } - } - - public VulnerableSoftware mapAffectedVersionToVulnerableSoftware(final QueryManager qm, final String vulnId, - String version, String purlStr, String cpeStr) { - version = StringUtils.trimToNull(version); - cpeStr = StringUtils.trimToNull(cpeStr); - purlStr = StringUtils.trimToNull(purlStr); - if (version == null || (cpeStr == null && purlStr == null)) { - return null; - } - - var vs = new VulnerableSoftware(); - if (purlStr != null) { - final PackageURL purl; - try { - purl = new PackageURL(purlStr); - vs = qm.getVulnerableSoftwareByPurlAndVersion(purl.getType(), purl.getNamespace(), purl.getName(), version); - if (vs != null) { - return vs; - } else { - vs = new VulnerableSoftware(); - vs.setPurlType(purl.getType()); - vs.setPurlNamespace(purl.getNamespace()); - vs.setPurlName(purl.getName()); - vs.setPurl(purl.canonicalize()); - vs.setVersion(version); - } - } catch (MalformedPackageURLException e) { - LOGGER.warn("Failed to parse PURL from \"%s\" for %s; Skipping".formatted(purlStr, vulnId), e); - return null; - } - } else { - try { - vs = qm.getVulnerableSoftwareByCpe23AndVersion(cpeStr, version); - if (vs != null) { - return vs; - } else { - vs = ModelConverter.convertCpe23UriToVulnerableSoftware(cpeStr); - vs.setVersion(version); - } - } catch (CpeParsingException | CpeEncodingException e) { - LOGGER.warn("Failed to parse CPE from \"%s\" for %s; Skipping".formatted(cpeStr, vulnId), e); - return null; - } - } - vs.setVulnerable(true); - return vs; - } - - public VulnerableSoftware mapAffectedRangeToVulnerableSoftware(final QueryManager qm, final String vulnId, - String range, String purlStr, String cpeStr) { - range = StringUtils.trimToNull(range); - cpeStr = StringUtils.trimToNull(cpeStr); - purlStr = StringUtils.trimToNull(purlStr); - if (range == null || (cpeStr == null && purlStr == null)) { - return null; - } - - final Vers vers; - try { - vers = Vers.parse(range); - } catch (VersException e) { - LOGGER.warn("Failed to parse vers range from \"%s\" for %s".formatted(range, vulnId), e); - return null; - } - - if (vers.constraints().isEmpty()) { - LOGGER.debug("Vers range \"%s\" (parsed: %s) for %s does not contain any constraints; Skipping".formatted(range, vers, vulnId)); - return null; - } else if (vers.constraints().size() > 2) { - // Vers ranges can express multiple "branches", which means that this method must potentially be able to return - // multiple `VulnerableSoftware`s, not just one. For example: - // vers:tomee/>=1.0.0-beta1|<=1.7.5|>=7.0.0-M1|<=7.0.7|>=7.1.0|<=7.1.2|>=8.0.0-M1|<=8.0.1 - // would result in the following branches: - // * vers:tomee/>=1.0.0-beta1|<=1.7.5 - // * vers:tomee/>=7.0.0-M1|<=7.0.7 - // * vers:tomee/>=7.1.0|<=7.1.2 - // * vers:tomee/>=8.0.0-M1|<=8.0.1 - // - // Branches are not always pairs, for example: - // vers:npm/1.2.3|>=2.0.0|<5.0.0 - // would result in: - // * vers:npm/1.2.3 - // * vers:npm/>=2.0.0|5.0.0 - // In both cases, separate `VulnerableSoftware`s are required. - // - // Because mirror-service does not currently produce such complex ranges, we log a warning - // and skip them if we encounter them. Occurrences of this log would indicate broken parsing - // logic in mirror-service. - LOGGER.warn("Vers range \"%s\" (parsed: %s) for %s contains more than two constraints; Skipping".formatted(range, vers, vulnId)); - return null; - } - - if (vers.constraints().size() == 1 && vers.constraints().get(0).comparator() == Comparator.WILDCARD) { - // Wildcards in VulnerableSoftware can be represented via either: - // * version=*, or - // * versionStartIncluding=0 - // We choose the more explicit first option. - // - // Also, as wildcards have the potential to lead to lots of false positives, - // we want to be informed when they enter our system. So logging a warning. - LOGGER.warn("Wildcard range %s was reported for %s".formatted(vers, vulnId)); - return mapAffectedVersionToVulnerableSoftware(qm, vulnId, "*", purlStr, cpeStr); - } - - String versionStartIncluding = null; - String versionStartExcluding = null; - String versionEndIncluding = null; - String versionEndExcluding = null; - - for (final Constraint constraint : vers.constraints()) { - if (constraint.version() == null - || constraint.version().equals("0") - || constraint.version().equals("*")) { - // Semantically, ">=0" is equivalent to versionStartIncluding=null, - // and ">0" is equivalent to versionStartExcluding=null. - // - // "<0", "<=0", and "=0" can be normalized to versionStartIncluding=null. - // - // "*" is a wildcard and can only be used on its own, without any comparator. - // The Vers parsing / validation performed above will thus fail for ranges like "vers:generic/>=*". - continue; - } - - switch (constraint.comparator()) { - case GREATER_THAN -> versionStartExcluding = constraint.version(); - case GREATER_THAN_OR_EQUAL -> versionStartIncluding = constraint.version(); - case LESS_THAN_OR_EQUAL -> versionEndIncluding = constraint.version(); - case LESS_THAN -> versionEndExcluding = constraint.version(); - default -> LOGGER.warn("Encountered unexpected comparator %s in %s for %s; Skipping" - .formatted(constraint.comparator(), vers, vulnId)); - } - } - - if (versionStartIncluding == null && versionStartExcluding == null - && versionEndIncluding == null && versionEndExcluding == null) { - LOGGER.warn("Unable to assemble a version range from %s for %s".formatted(vers, vulnId)); - return null; - } - if ((versionStartIncluding != null || versionStartExcluding != null) - && (versionEndIncluding == null && versionEndExcluding == null)) { - LOGGER.warn("Skipping indefinite version range assembled from %s for %s".formatted(vers, vulnId)); - return null; - } - - VulnerableSoftware vs; - if (purlStr != null) { - final PackageURL purl; - try { - purl = new PackageURL(purlStr); - } catch (MalformedPackageURLException e) { - LOGGER.warn("Failed to parse PURL from \"%s\" for %s; Skipping".formatted(purlStr, vulnId), e); - return null; - } - vs = qm.getVulnerableSoftwareByPurl(purl.getType(), purl.getNamespace(), purl.getName(), - versionEndExcluding, versionEndIncluding, null, versionStartIncluding); - if (vs != null) { - return vs; - } - vs = new VulnerableSoftware(); - vs.setPurlType(purl.getType()); - vs.setPurlNamespace(purl.getNamespace()); - vs.setPurlName(purl.getName()); - vs.setPurl(purl.canonicalize()); - } else { - vs = qm.getVulnerableSoftwareByCpe23(cpeStr, versionEndExcluding, - versionEndIncluding, versionStartExcluding, versionStartIncluding); - if (vs != null) { - return vs; - } - try { - vs = ModelConverter.convertCpe23UriToVulnerableSoftware(cpeStr); - } catch (CpeParsingException | CpeEncodingException e) { - LOGGER.warn("Failed to parse CPE from \"%s\" for %s; Skipping".formatted(cpeStr, vulnId), e); - return null; - } - } - - vs.setVulnerable(true); - vs.setVersionStartExcluding(versionStartExcluding); - vs.setVersionStartIncluding(versionStartIncluding); - vs.setVersionEndExcluding(versionEndExcluding); - vs.setVersionEndIncluding(versionEndIncluding); - return vs; - } -} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 0dd346c5f..8ea6b180f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -435,7 +435,7 @@ kafka.streams.transient.processing.exception.threshold.count=50 kafka.streams.transient.processing.exception.threshold.interval=PT30M # Optional -# alpine.kafka.processor..processing.order=key +# alpine.kafka.processor..processing.order=partition # alpine.kafka.processor..max.batch.size= # alpine.kafka.processor..max.concurrency=1 # alpine.kafka.processor..retry.initial.delay.ms=1000 @@ -444,10 +444,10 @@ kafka.streams.transient.processing.exception.threshold.interval=PT30M # alpine.kafka.processor..retry.max.delay.ms=60000 # alpine.kafka.processor..consumer.= alpine.kafka.processor.repo-meta-result.max.concurrency=3 -alpine.kafka.processor.repo-meta-result.processing.order=key +alpine.kafka.processor.repo-meta-result.processing.order=partition alpine.kafka.processor.repo-meta-result.consumer.group.id=dtrack-apiserver-processor alpine.kafka.processor.vuln-mirror.max.concurrency=3 -alpine.kafka.processor.vuln-mirror.processing.order=key +alpine.kafka.processor.vuln-mirror.processing.order=partition alpine.kafka.processor.vuln-mirror.retry.initial.delay.ms=3000 alpine.kafka.processor.vuln-mirror.retry.multiplier=2 alpine.kafka.processor.vuln-mirror.retry.randomization.factor=0.3 diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java deleted file mode 100644 index 2a5deb598..000000000 --- a/src/test/java/org/dependencytrack/event/kafka/processor/KafkaProcessorTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.dependencytrack.event.kafka.processor; - - -import net.mguenther.kafka.junit.ExternalKafkaCluster; -import net.mguenther.kafka.junit.KeyValue; -import net.mguenther.kafka.junit.ReadKeyValues; -import net.mguenther.kafka.junit.TopicConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.common.serialization.BooleanDeserializer; -import org.apache.kafka.common.serialization.IntegerDeserializer; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.testcontainers.redpanda.RedpandaContainer; -import org.testcontainers.utility.DockerImageName; - -import java.time.Duration; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -public class KafkaProcessorTest { - - @Rule - public RedpandaContainer container = new RedpandaContainer(DockerImageName - .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); - - ExternalKafkaCluster kafka; - - @Before - public void setUp() { - kafka = ExternalKafkaCluster.at(container.getBootstrapServers()); - } - - @Test - public void testSingleRecordProcessor() throws Exception { - kafka.createTopic(TopicConfig.withName("input")); - kafka.createTopic(TopicConfig.withName("output")); - -// final RecordProcessor recordProcessor = -// record -> List.of(new ProducerRecord<>("output", 2, true)); -// -// final var processorFactory = new KafkaRecordConsumerFactory(); -// try (final KafkaProcessor processor = processorFactory.createProcessor(recordProcessor)) { -// processor.start(); -// -// kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", 123L))) -// .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) -// .with(VALUE_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName())); -// } - - await() - .atMost(Duration.ofSeconds(5)) - .untilAsserted(() -> { - final List> records = kafka.read(ReadKeyValues.from("output", Integer.class, Boolean.class) - .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, IntegerDeserializer.class.getName()) - .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BooleanDeserializer.class.getName())); - - assertThat(records).hasSize(1); - }); - } - -} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java similarity index 88% rename from src/test/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java index b8a57778c..71e972352 100644 --- a/src/test/java/org/dependencytrack/event/kafka/streams/processor/MirrorVulnerabilityProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java @@ -1,55 +1,78 @@ -package org.dependencytrack.event.kafka.streams.processor; +package org.dependencytrack.event.kafka.processor; -import org.apache.kafka.common.serialization.Serdes; +import alpine.Config; +import net.mguenther.kafka.junit.ExternalKafkaCluster; +import net.mguenther.kafka.junit.KeyValue; +import net.mguenther.kafka.junit.SendKeyValues; +import net.mguenther.kafka.junit.TopicConfig; import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.TestInputTopic; -import org.apache.kafka.streams.TopologyTestDriver; -import org.apache.kafka.streams.kstream.Consumed; -import org.cyclonedx.proto.v1_4.Bom; import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerde; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.event.kafka.processor.api.RecordProcessorManager; import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; import org.dependencytrack.model.Severity; import org.dependencytrack.model.Vulnerability; import org.dependencytrack.persistence.CweImporter; -import org.dependencytrack.util.KafkaTestUtil; import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.testcontainers.redpanda.RedpandaContainer; +import org.testcontainers.utility.DockerImageName; +import java.time.Duration; +import java.util.List; +import java.util.Objects; + +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.dependencytrack.util.KafkaTestUtil.generateBomFromJson; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class VulnerabilityMirrorProcessorTest extends PersistenceCapableTest { -public class MirrorVulnerabilityProcessorTest extends PersistenceCapableTest { + @Rule + public RedpandaContainer kafkaContainer = new RedpandaContainer(DockerImageName + .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); - private TopologyTestDriver testDriver; - private TestInputTopic inputTopic; + private ExternalKafkaCluster kafka; + private RecordProcessorManager processorManager; @Before public void setUp() throws Exception { - final var streamsBuilder = new StreamsBuilder(); - streamsBuilder - .stream("input-topic", Consumed - .with(Serdes.String(), new KafkaProtobufSerde<>(Bom.parser()))) - .process(MirrorVulnerabilityProcessor::new); + new CweImporter().processCweDefinitions(); // Required for CWE mapping - testDriver = new TopologyTestDriver(streamsBuilder.build()); - inputTopic = testDriver.createInputTopic("input-topic", - new StringSerializer(), new KafkaProtobufSerializer<>()); + kafka = ExternalKafkaCluster.at(kafkaContainer.getBootstrapServers()); - new CweImporter().processCweDefinitions(); // Required for CWE mapping + kafka.createTopic(TopicConfig + .withName(KafkaTopics.NEW_VULNERABILITY.name()) + .withNumberOfPartitions(3)); + + final var configMock = mock(Config.class); + when(configMock.getProperty(eq(ConfigKey.KAFKA_BOOTSTRAP_SERVERS))) + .thenReturn(kafkaContainer.getBootstrapServers()); + + processorManager = new RecordProcessorManager(configMock); + processorManager.register(VulnerabilityMirrorProcessor.PROCESSOR_NAME, + new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); + processorManager.startAll(); } @After public void tearDown() { - if (testDriver != null) { - testDriver.close(); + if (processorManager != null) { + processorManager.close(); } } @Test public void testProcessNvdVuln() throws Exception { - inputTopic.pipeInput("NVD/CVE-2022-40489", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -91,9 +114,17 @@ public void testProcessNvdVuln() throws Exception { { "url": "https://github.com/thinkcmf/thinkcmf/issues/736" } ] } - """)); + """; + + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); - final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -160,7 +191,7 @@ public void testProcessNvdVuln() throws Exception { @Test public void testProcessGitHubVuln() throws Exception { - inputTopic.pipeInput("GITHUB/GHSA-fxwm-579q-49qq", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -223,9 +254,17 @@ public void testProcessGitHubVuln() throws Exception { { "url": "https://github.com/advisories/GHSA-fxwm-579q-49qq" } ] } - """)); + """; - final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-fxwm-579q-49qq"); + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("GITHUB/GHSA-fxwm-579q-49qq", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("GITHUB", "GHSA-fxwm-579q-49qq"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("GHSA-fxwm-579q-49qq"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -375,7 +414,7 @@ public void testProcessGitHubVuln() throws Exception { @Test public void testProcessOsvVuln() throws Exception { - inputTopic.pipeInput("OSV/GHSA-2cc5-23r7-vc4v", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -427,9 +466,17 @@ public void testProcessOsvVuln() throws Exception { { "url": "https://github.com/ratpack/ratpack/blob/29434f7ac6fd4b36a4495429b70f4c8163100332/ratpack-session/src/main/java/ratpack/session/clientside/ClientSideSessionConfig.java#L29" } ] } - """)); + """; + + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("OSV/GHSA-2cc5-23r7-vc4v", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); - final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-2cc5-23r7-vc4v"); + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("GITHUB", "GHSA-2cc5-23r7-vc4v"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("GHSA-2cc5-23r7-vc4v"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -555,7 +602,7 @@ public void testProcessOsvVuln() throws Exception { @Test public void testProcessVulnWithoutAffects() throws Exception { - inputTopic.pipeInput("NVD/CVE-2022-40489", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -573,9 +620,17 @@ public void testProcessVulnWithoutAffects() throws Exception { } ] } - """)); + """; + + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); - final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -613,7 +668,7 @@ public void testProcessVulnWithoutAffects() throws Exception { @Test public void testProcessVulnWithUnmatchedAffectsBomRef() throws Exception { - inputTopic.pipeInput("NVD/CVE-2022-40489", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -639,9 +694,17 @@ public void testProcessVulnWithUnmatchedAffectsBomRef() throws Exception { } ] } - """)); + """; - final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -679,7 +742,7 @@ public void testProcessVulnWithUnmatchedAffectsBomRef() throws Exception { @Test public void testProcessVulnWithVersConstraints() throws Exception { - inputTopic.pipeInput("NVD/CVE-2022-40489", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -731,9 +794,17 @@ public void testProcessVulnWithVersConstraints() throws Exception { } ] } - """)); + """; + + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); - final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -935,7 +1006,7 @@ public void testProcessVulnWithVersConstraints() throws Exception { @Test public void testProcessVulnWithInvalidCpeOrPurl() throws Exception { - inputTopic.pipeInput("NVD/CVE-2022-40489", KafkaTestUtil.generateBomFromJson(""" + final var bovJson = """ { "components": [ { @@ -997,9 +1068,17 @@ public void testProcessVulnWithInvalidCpeOrPurl() throws Exception { } ] } - """)); + """; + + kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( + new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) + )) + .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); - final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); + final Vulnerability vuln = await("Processed Vulnerability") + .atMost(Duration.ofSeconds(5)) + .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -1035,4 +1114,4 @@ public void testProcessVulnWithInvalidCpeOrPurl() throws Exception { assertThat(vuln.getVulnerableSoftware()).isEmpty(); } -} +} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java index ea856ba65..47d4e1456 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java @@ -37,8 +37,8 @@ public class RecordProcessorManagerTest { public RedpandaContainer kafkaContainer = new RedpandaContainer(DockerImageName .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); - ExternalKafkaCluster kafka; - Config configMock; + private ExternalKafkaCluster kafka; + private Config configMock; @Before public void setUp() { diff --git a/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java b/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java deleted file mode 100644 index 1d3fb7cf9..000000000 --- a/src/test/java/org/dependencytrack/event/kafka/streams/KafkaStreamsTest.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.dependencytrack.event.kafka.streams; - -import net.mguenther.kafka.junit.ExternalKafkaCluster; -import net.mguenther.kafka.junit.TopicConfig; -import org.apache.kafka.streams.KafkaStreams; -import org.apache.kafka.streams.StreamsConfig; -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.kafka.KafkaTopics; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.testcontainers.redpanda.RedpandaContainer; -import org.testcontainers.utility.DockerImageName; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; - -import static org.dependencytrack.assertion.Assertions.assertConditionWithTimeout; - -abstract class KafkaStreamsTest extends PersistenceCapableTest { - - @Rule - public RedpandaContainer container = new RedpandaContainer(DockerImageName - .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); - - KafkaStreams kafkaStreams; - ExternalKafkaCluster kafka; - private Path kafkaStreamsStateDirectory; - - @Before - public void setUp() throws Exception { - kafka = ExternalKafkaCluster.at(container.getBootstrapServers()); - - kafka.createTopic(TopicConfig - .withName(KafkaTopics.VULN_ANALYSIS_COMMAND.name()) - .withNumberOfPartitions(3) - .withNumberOfReplicas(1)); - kafka.createTopic(TopicConfig - .withName(KafkaTopics.VULN_ANALYSIS_RESULT.name()) - .withNumberOfPartitions(3) - .withNumberOfReplicas(1)); - kafka.createTopic(TopicConfig - .withName(KafkaTopics.REPO_META_ANALYSIS_RESULT.name()) - .withNumberOfPartitions(3) - .withNumberOfReplicas(1)); - kafka.createTopic(TopicConfig - .withName(KafkaTopics.NEW_VULNERABILITY.name()) - .withNumberOfPartitions(3) - .withNumberOfReplicas(1)); - - kafkaStreamsStateDirectory = Files.createTempDirectory(getClass().getSimpleName()); - - final var streamsConfig = KafkaStreamsInitializer.getDefaultProperties(); - streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, container.getBootstrapServers()); - streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, getClass().getSimpleName()); - streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, kafkaStreamsStateDirectory.toString()); - streamsConfig.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, "3"); - - kafkaStreams = new KafkaStreams(new KafkaStreamsTopologyFactory().createTopology(), streamsConfig); - kafkaStreams.start(); - - assertConditionWithTimeout(() -> KafkaStreams.State.RUNNING == kafkaStreams.state(), Duration.ofSeconds(5)); - } - - @After - public void tearDown() { - if (kafkaStreams != null) { - kafkaStreams.close(); - } - if (kafkaStreamsStateDirectory != null) { - kafkaStreamsStateDirectory.toFile().delete(); - } - } - -} From 2a5e698ddcf63873e74caa72125ab4298b424560 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 22:14:57 +0100 Subject: [PATCH 13/26] WIP: Migrate `RepositoryMetaResultProcessor` Signed-off-by: nscuro --- .../RepositoryMetaResultProcessor.java | 2 +- .../RepositoryMetaResultProcessor.java | 204 ------------------ .../processor/AbstractProcessorTest.java | 22 ++ .../RepositoryMetaResultProcessorTest.java | 128 +++++------ .../VulnerabilityMirrorProcessorTest.java | 135 +++--------- 5 files changed, 105 insertions(+), 386 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessor.java create mode 100644 src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java rename src/test/java/org/dependencytrack/event/kafka/{streams => }/processor/RepositoryMetaResultProcessorTest.java (89%) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index 1e7ca5e83..bb6d716d9 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -34,7 +34,7 @@ public class RepositoryMetaResultProcessor implements SingleRecordProcessor { - - private static final Logger LOGGER = Logger.getLogger(RepositoryMetaResultProcessor.class); - private static final Timer TIMER = Timer.builder("repo_meta_result_processing") - .description("Time taken to process repository meta analysis results") - .register(Metrics.getRegistry()); - - @Override - public void process(final Record record) { - final Timer.Sample timerSample = Timer.start(); - if (!isRecordValid(record)) { - return; - } - try (final var qm = new QueryManager()) { - synchronizeRepositoryMetadata(qm, record); - IntegrityMetaComponent integrityMetaComponent = synchronizeIntegrityMetadata(qm, record); - if (integrityMetaComponent != null) { - performIntegrityCheck(integrityMetaComponent, record.value(), qm); - } - } catch (Exception e) { - LOGGER.error("An unexpected error occurred while processing record %s".formatted(record), e); - } finally { - timerSample.stop(TIMER); - } - } - - private IntegrityMetaComponent synchronizeIntegrityMetadata(final QueryManager queryManager, final Record record) throws MalformedPackageURLException { - final AnalysisResult result = record.value(); - PackageURL purl = new PackageURL(result.getComponent().getPurl()); - if (result.hasIntegrityMeta()) { - return synchronizeIntegrityMetaResult(record, queryManager, purl); - } else { - LOGGER.debug("Incoming result for component with purl %s does not include component integrity info".formatted(purl)); - return null; - } - } - - private void synchronizeRepositoryMetadata(final QueryManager queryManager, final Record record) throws Exception { - PersistenceManager pm = queryManager.getPersistenceManager(); - final AnalysisResult result = record.value(); - PackageURL purl = new PackageURL(result.getComponent().getPurl()); - - // It is possible that the same meta info is reported for multiple components in parallel, - // causing unique constraint violations when attempting to insert into the REPOSITORY_META_COMPONENT table. - // In such cases, we can get away with simply retrying to SELECT+UPDATE or INSERT again. We'll attempt - // up to 3 times before giving up. - for (int i = 0; i < 3; i++) { - final Transaction trx = pm.currentTransaction(); - try { - RepositoryMetaComponent repositoryMetaComponentResult = createRepositoryMetaResult(record, pm, purl); - if (repositoryMetaComponentResult != null) { - trx.begin(); - pm.makePersistent(repositoryMetaComponentResult); - trx.commit(); - break; // this means that transaction was successful and we do not need to retry - } - } catch (JDODataStoreException e) { - // TODO: DataNucleus doesn't map constraint violation exceptions very well, - // so we have to depend on the exception of the underlying JDBC driver to - // tell us what happened. We currently only handle PostgreSQL, but we'll have - // to do the same for at least H2 and MSSQL. - if (ExceptionUtils.getRootCause(e) instanceof final SQLException se - && PSQLState.UNIQUE_VIOLATION.getState().equals(se.getSQLState())) { - continue; // Retry - } - - throw e; - } finally { - if (trx.isActive()) { - trx.rollback(); - } - } - } - } - - private RepositoryMetaComponent createRepositoryMetaResult(Record incomingAnalysisResultRecord, PersistenceManager pm, PackageURL purl) throws Exception { - final AnalysisResult result = incomingAnalysisResultRecord.value(); - if (result.hasLatestVersion()) { - try (final Query query = pm.newQuery(RepositoryMetaComponent.class)) { - query.setFilter("repositoryType == :repositoryType && namespace == :namespace && name == :name"); - query.setParameters( - RepositoryType.resolve(purl), - purl.getNamespace(), - purl.getName() - ); - RepositoryMetaComponent persistentRepoMetaComponent = query.executeUnique(); - if (persistentRepoMetaComponent == null) { - persistentRepoMetaComponent = new RepositoryMetaComponent(); - } - - if (persistentRepoMetaComponent.getLastCheck() != null - && persistentRepoMetaComponent.getLastCheck().after(new Date(incomingAnalysisResultRecord.timestamp()))) { - LOGGER.warn(""" - Received repository meta information for %s that is older\s - than what's already in the database; Discarding - """.formatted(purl)); - return null; - } - - persistentRepoMetaComponent.setRepositoryType(RepositoryType.resolve(purl)); - persistentRepoMetaComponent.setNamespace(purl.getNamespace()); - persistentRepoMetaComponent.setName(purl.getName()); - if (result.hasLatestVersion()) { - persistentRepoMetaComponent.setLatestVersion(result.getLatestVersion()); - } - if (result.hasPublished()) { - persistentRepoMetaComponent.setPublished(new Date(result.getPublished().getSeconds() * 1000)); - } - persistentRepoMetaComponent.setLastCheck(new Date(incomingAnalysisResultRecord.timestamp())); - return persistentRepoMetaComponent; - } - } else { - return null; - } - } - - private IntegrityMetaComponent synchronizeIntegrityMetaResult(final Record incomingAnalysisResultRecord, QueryManager queryManager, PackageURL purl) { - final AnalysisResult result = incomingAnalysisResultRecord.value(); - IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(purl.toString()); - if (persistentIntegrityMetaComponent != null && persistentIntegrityMetaComponent.getStatus() != null && persistentIntegrityMetaComponent.getStatus().equals(FetchStatus.PROCESSED)) { - LOGGER.warn(""" - Received hash information for %s that has already been processed; Discarding - """.formatted(purl)); - return persistentIntegrityMetaComponent; - } - if (persistentIntegrityMetaComponent == null) { - persistentIntegrityMetaComponent = new IntegrityMetaComponent(); - } - - if (result.getIntegrityMeta().hasMd5() || result.getIntegrityMeta().hasSha1() || result.getIntegrityMeta().hasSha256() - || result.getIntegrityMeta().hasSha512() || result.getIntegrityMeta().hasCurrentVersionLastModified()) { - Optional.ofNullable(result.getIntegrityMeta().getMd5()).ifPresent(persistentIntegrityMetaComponent::setMd5); - Optional.ofNullable(result.getIntegrityMeta().getSha1()).ifPresent(persistentIntegrityMetaComponent::setSha1); - Optional.ofNullable(result.getIntegrityMeta().getSha256()).ifPresent(persistentIntegrityMetaComponent::setSha256); - Optional.ofNullable(result.getIntegrityMeta().getSha512()).ifPresent(persistentIntegrityMetaComponent::setSha512); - persistentIntegrityMetaComponent.setPurl(result.getComponent().getPurl()); - persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl()); - persistentIntegrityMetaComponent.setPublishedAt(result.getIntegrityMeta().hasCurrentVersionLastModified() ? new Date(result.getIntegrityMeta().getCurrentVersionLastModified().getSeconds() * 1000) : null); - persistentIntegrityMetaComponent.setStatus(FetchStatus.PROCESSED); - } else { - persistentIntegrityMetaComponent.setMd5(null); - persistentIntegrityMetaComponent.setSha256(null); - persistentIntegrityMetaComponent.setSha1(null); - persistentIntegrityMetaComponent.setSha512(null); - persistentIntegrityMetaComponent.setPurl(purl.toString()); - persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl()); - persistentIntegrityMetaComponent.setStatus(FetchStatus.NOT_AVAILABLE); - } - return queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent); - } - - private static boolean isRecordValid(final Record record) { - final AnalysisResult result = record.value(); - if (!result.hasComponent()) { - LOGGER.warn(""" - Received repository meta information without component,\s - will not be able to correlate; Dropping - """); - return false; - } - - try { - new PackageURL(result.getComponent().getPurl()); - } catch (MalformedPackageURLException e) { - LOGGER.warn(""" - Received repository meta information with invalid PURL,\s - will not be able to correlate; Dropping - """, e); - return false; - } - return true; - } -} diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java new file mode 100644 index 000000000..44db29226 --- /dev/null +++ b/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java @@ -0,0 +1,22 @@ +package org.dependencytrack.event.kafka.processor; + +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.record.TimestampType; +import org.dependencytrack.AbstractPostgresEnabledTest; + +import java.time.Instant; +import java.util.Optional; + +abstract class AbstractProcessorTest extends AbstractPostgresEnabledTest { + + ConsumerRecord createConsumerRecord(final K key, final V value) { + return createConsumerRecord(key, value, Instant.now()); + } + + ConsumerRecord createConsumerRecord(final K key, final V value, final Instant timestamp) { + return new ConsumerRecord<>("topic", 0, 1, timestamp.toEpochMilli(), TimestampType.CREATE_TIME, -1, -1, + key, value, new RecordHeaders(), Optional.empty()); + } + +} diff --git a/src/test/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java similarity index 89% rename from src/test/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java index 5f14df992..e763c8546 100644 --- a/src/test/java/org/dependencytrack/event/kafka/streams/processor/RepositoryMetaResultProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java @@ -1,15 +1,6 @@ -package org.dependencytrack.event.kafka.streams.processor; +package org.dependencytrack.event.kafka.processor; import com.google.protobuf.Timestamp; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.streams.TestInputTopic; -import org.apache.kafka.streams.Topology; -import org.apache.kafka.streams.TopologyTestDriver; -import org.apache.kafka.streams.test.TestRecord; -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; import org.dependencytrack.model.Component; import org.dependencytrack.model.FetchStatus; import org.dependencytrack.model.IntegrityAnalysis; @@ -19,7 +10,6 @@ import org.dependencytrack.model.RepositoryType; import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult; import org.dependencytrack.proto.repometaanalysis.v1.IntegrityMeta; -import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,37 +24,20 @@ import static org.assertj.core.api.Assertions.assertThat; -public class RepositoryMetaResultProcessorTest extends PersistenceCapableTest { +public class RepositoryMetaResultProcessorTest extends AbstractProcessorTest { @Rule public EnvironmentVariables environmentVariables = new EnvironmentVariables(); - private TopologyTestDriver testDriver; - private TestInputTopic inputTopic; - @Before - public void setUp() { - environmentVariables.set("INTEGRITY_CHECK_ENABLED", "true"); - final var topology = new Topology(); - topology.addSource("sourceProcessor", - new StringDeserializer(), new KafkaProtobufDeserializer<>(AnalysisResult.parser()), "input-topic"); - topology.addProcessor("metaResultProcessor", - RepositoryMetaResultProcessor::new, "sourceProcessor"); - - testDriver = new TopologyTestDriver(topology); - inputTopic = testDriver.createInputTopic("input-topic", - new StringSerializer(), new KafkaProtobufSerializer<>()); - } + public void setUp() throws Exception { + super.setUp(); - @After - public void tearDown() { - if (testDriver != null) { - testDriver.close(); - } + environmentVariables.set("INTEGRITY_CHECK_ENABLED", "true"); } @Test - public void processNewMetaModelTest() { + public void processNewMetaModelTest() throws Exception { final var published = Instant.now().minus(5, ChronoUnit.MINUTES); final var result = AnalysisResult.newBuilder() @@ -75,10 +48,10 @@ public void processNewMetaModelTest() { .setSeconds(published.getEpochSecond())) .build(); - inputTopic.pipeInput("pkg:maven/foo/bar", result); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); - final RepositoryMetaComponent metaComponent = - qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); + final RepositoryMetaComponent metaComponent = qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); assertThat(metaComponent).isNotNull(); assertThat(metaComponent.getRepositoryType()).isEqualTo(RepositoryType.MAVEN); assertThat(metaComponent.getNamespace()).isEqualTo("foo"); @@ -88,14 +61,15 @@ public void processNewMetaModelTest() { } @Test - public void processWithoutComponentDetailsTest() { + public void processWithoutComponentDetailsTest() throws Exception { final var result = AnalysisResult.newBuilder() .setLatestVersion("1.2.4") .setPublished(Timestamp.newBuilder() .setSeconds(Instant.now().getEpochSecond())) .build(); - inputTopic.pipeInput("foo", result); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); final Query query = qm.getPersistenceManager().newQuery(RepositoryMetaComponent.class); query.setResult("count(this)"); @@ -104,7 +78,7 @@ public void processWithoutComponentDetailsTest() { } @Test - public void processUpdateExistingMetaModelTest() { + public void processUpdateExistingMetaModelTest() throws Exception { final var metaComponent = new RepositoryMetaComponent(); metaComponent.setRepositoryType(RepositoryType.MAVEN); metaComponent.setNamespace("foo"); @@ -124,7 +98,8 @@ public void processUpdateExistingMetaModelTest() { .setSeconds(published.getEpochSecond())) .build(); - inputTopic.pipeInput("pkg:maven/foo/bar", result); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); qm.getPersistenceManager().refresh(metaComponent); assertThat(metaComponent).isNotNull(); @@ -136,7 +111,7 @@ public void processUpdateExistingMetaModelTest() { } @Test - public void processUpdateOutOfOrderMetaModelTest() { + public void processUpdateOutOfOrderMetaModelTest() throws Exception { final var testStartTime = new Date(); final var metaComponent = new RepositoryMetaComponent(); @@ -159,7 +134,8 @@ public void processUpdateOutOfOrderMetaModelTest() { .build(); // Pipe in a record that was produced 10 seconds ago, 5 seconds before metaComponent's lastCheck. - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now().minusSeconds(10))); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result, Instant.now().minusSeconds(10))); qm.getPersistenceManager().refresh(metaComponent); assertThat(metaComponent).isNotNull(); @@ -172,7 +148,7 @@ public void processUpdateOutOfOrderMetaModelTest() { } @Test - public void processUpdateIntegrityResultTest() { + public void processUpdateIntegrityResultTest() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -204,7 +180,8 @@ public void processUpdateIntegrityResultTest() { .setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -225,7 +202,7 @@ public void processUpdateIntegrityResultTest() { } @Test - public void testIntegrityCheckWhenComponentHashIsMissing() { + public void testIntegrityCheckWhenComponentHashIsMissing() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -256,7 +233,8 @@ public void testIntegrityCheckWhenComponentHashIsMissing() { .setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -275,7 +253,7 @@ public void testIntegrityCheckWhenComponentHashIsMissing() { } @Test - public void testIntegrityAnalysisWillNotBePerformedIfNoIntegrityDataInResult() { + public void testIntegrityAnalysisWillNotBePerformedIfNoIntegrityDataInResult() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -306,14 +284,15 @@ public void testIntegrityAnalysisWillNotBePerformedIfNoIntegrityDataInResult() { .setPurl("pkg:maven/foo/bar@1.2.3")) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNull(); } @Test - public void testIntegrityCheckWillNotBeDoneIfComponentUuidAndIntegrityDataIsMissing() { + public void testIntegrityCheckWillNotBeDoneIfComponentUuidAndIntegrityDataIsMissing() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -344,14 +323,15 @@ public void testIntegrityCheckWillNotBeDoneIfComponentUuidAndIntegrityDataIsMiss //component uuid has not been set .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNull(); } @Test - public void testIntegrityIfResultHasIntegrityDataAndComponentUuidIsMissing() { + public void testIntegrityIfResultHasIntegrityDataAndComponentUuidIsMissing() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -383,7 +363,8 @@ public void testIntegrityIfResultHasIntegrityDataAndComponentUuidIsMissing() { //component uuid has not been set .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNotNull(); @@ -393,8 +374,7 @@ public void testIntegrityIfResultHasIntegrityDataAndComponentUuidIsMissing() { @Test - public void testIntegrityCheckWillNotBeDoneIfComponentIsNotInDb() { - + public void testIntegrityCheckWillNotBeDoneIfComponentIsNotInDb() throws Exception { UUID uuid = UUID.randomUUID(); var integrityMetaComponent = new IntegrityMetaComponent(); @@ -413,14 +393,15 @@ public void testIntegrityCheckWillNotBeDoneIfComponentIsNotInDb() { .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(uuid); assertThat(analysis).isNull(); } @Test - public void testIntegrityCheckShouldReturnComponentHashMissing() { + public void testIntegrityCheckShouldReturnComponentHashMissing() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -450,7 +431,8 @@ public void testIntegrityCheckShouldReturnComponentHashMissing() { .setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -469,7 +451,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissing() { } @Test - public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() { + public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -498,7 +480,8 @@ public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() .setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -515,7 +498,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() } @Test - public void testIntegrityCheckShouldFailIfNoHashMatch() { + public void testIntegrityCheckShouldFailIfNoHashMatch() throws Exception { // Create an active project with one component. final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); final var componentProjectA = new Component(); @@ -548,7 +531,8 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() { .setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -567,7 +551,7 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() { } @Test - public void processUpdateIntegrityResultNotAvailableTest() { + public void processUpdateIntegrityResultNotAvailableTest() throws Exception { var integrityMetaComponent = new IntegrityMetaComponent(); integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); integrityMetaComponent.setStatus(FetchStatus.IN_PROGRESS); @@ -582,7 +566,8 @@ public void processUpdateIntegrityResultNotAvailableTest() { .setIntegrityMeta(IntegrityMeta.newBuilder().setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -594,8 +579,7 @@ public void processUpdateIntegrityResultNotAvailableTest() { } @Test - public void processUpdateOldIntegrityResultSent() { - + public void processUpdateOldIntegrityResultSent() throws Exception { Date date = Date.from(Instant.now().minus(15, ChronoUnit.MINUTES)); var integrityMetaComponent = new IntegrityMetaComponent(); integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); @@ -613,7 +597,8 @@ public void processUpdateOldIntegrityResultSent() { .setSha1("a94a8fe5ccb19ba61c4c0873d391e587982fbbd3").setMetaSourceUrl("test2").build()) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -626,7 +611,7 @@ public void processUpdateOldIntegrityResultSent() { @Test - public void processBothMetaModelAndIntegrityMeta() { + public void processBothMetaModelAndIntegrityMeta() throws Exception { final var published = Instant.now().minus(5, ChronoUnit.MINUTES); var integrityMetaComponent = new IntegrityMetaComponent(); integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); @@ -646,7 +631,8 @@ public void processBothMetaModelAndIntegrityMeta() { .setMetaSourceUrl("test").build()) .build(); - inputTopic.pipeInput("pkg:maven/foo/bar", result); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); final RepositoryMetaComponent metaComponent = qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); @@ -666,7 +652,7 @@ public void processBothMetaModelAndIntegrityMeta() { } @Test - public void processUpdateIntegrityResultNotSentTest() { + public void processUpdateIntegrityResultNotSentTest() throws Exception { var integrityMetaComponent = new IntegrityMetaComponent(); integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); integrityMetaComponent.setStatus(FetchStatus.IN_PROGRESS); @@ -680,7 +666,8 @@ public void processUpdateIntegrityResultNotSentTest() { .setPurl("pkg:maven/foo/bar@1.2.3")) .build(); - inputTopic.pipeInput(new TestRecord<>("pkg:maven/foo/bar@1.2.3", result, Instant.now())); + final var processor = new RepositoryMetaResultProcessor(); + processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -690,4 +677,5 @@ public void processUpdateIntegrityResultNotSentTest() { assertThat(integrityMetaComponent.getLastFetch()).isEqualTo(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); } + } \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java index 71e972352..701eddd1b 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java @@ -1,73 +1,21 @@ package org.dependencytrack.event.kafka.processor; -import alpine.Config; -import net.mguenther.kafka.junit.ExternalKafkaCluster; -import net.mguenther.kafka.junit.KeyValue; -import net.mguenther.kafka.junit.SendKeyValues; -import net.mguenther.kafka.junit.TopicConfig; -import org.apache.kafka.common.serialization.StringSerializer; -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.event.kafka.processor.api.RecordProcessorManager; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; import org.dependencytrack.model.Severity; import org.dependencytrack.model.Vulnerability; import org.dependencytrack.persistence.CweImporter; -import org.junit.After; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.testcontainers.redpanda.RedpandaContainer; -import org.testcontainers.utility.DockerImageName; -import java.time.Duration; -import java.util.List; -import java.util.Objects; - -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; import static org.dependencytrack.util.KafkaTestUtil.generateBomFromJson; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class VulnerabilityMirrorProcessorTest extends PersistenceCapableTest { - - @Rule - public RedpandaContainer kafkaContainer = new RedpandaContainer(DockerImageName - .parse("docker.redpanda.com/vectorized/redpanda:v23.2.13")); - private ExternalKafkaCluster kafka; - private RecordProcessorManager processorManager; +public class VulnerabilityMirrorProcessorTest extends AbstractProcessorTest { @Before public void setUp() throws Exception { - new CweImporter().processCweDefinitions(); // Required for CWE mapping - - kafka = ExternalKafkaCluster.at(kafkaContainer.getBootstrapServers()); + super.setUp(); - kafka.createTopic(TopicConfig - .withName(KafkaTopics.NEW_VULNERABILITY.name()) - .withNumberOfPartitions(3)); - - final var configMock = mock(Config.class); - when(configMock.getProperty(eq(ConfigKey.KAFKA_BOOTSTRAP_SERVERS))) - .thenReturn(kafkaContainer.getBootstrapServers()); - - processorManager = new RecordProcessorManager(configMock); - processorManager.register(VulnerabilityMirrorProcessor.PROCESSOR_NAME, - new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); - processorManager.startAll(); - } - - @After - public void tearDown() { - if (processorManager != null) { - processorManager.close(); - } + new CweImporter().processCweDefinitions(); // Required for CWE mapping } @Test @@ -116,15 +64,10 @@ public void testProcessNvdVuln() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -256,15 +199,10 @@ public void testProcessGitHubVuln() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("GITHUB/GHSA-fxwm-579q-49qq", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("GITHUB/GHSA-fxwm-579q-49qq", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("GITHUB", "GHSA-fxwm-579q-49qq"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-fxwm-579q-49qq"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("GHSA-fxwm-579q-49qq"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -468,15 +406,10 @@ public void testProcessOsvVuln() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("OSV/GHSA-2cc5-23r7-vc4v", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("OSV/GHSA-2cc5-23r7-vc4v", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("GITHUB", "GHSA-2cc5-23r7-vc4v"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-2cc5-23r7-vc4v"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("GHSA-2cc5-23r7-vc4v"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -622,15 +555,10 @@ public void testProcessVulnWithoutAffects() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -696,15 +624,10 @@ public void testProcessVulnWithUnmatchedAffectsBomRef() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -796,15 +719,10 @@ public void testProcessVulnWithVersConstraints() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); @@ -1070,15 +988,10 @@ public void testProcessVulnWithInvalidCpeOrPurl() throws Exception { } """; - kafka.send(SendKeyValues.to(KafkaTopics.NEW_VULNERABILITY.name(), List.of( - new KeyValue<>("NVD/CVE-2022-40489", generateBomFromJson(bovJson)) - )) - .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(VALUE_SERIALIZER_CLASS_CONFIG, KafkaProtobufSerializer.class.getName())); + final var processor = new VulnerabilityMirrorProcessor(); + processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); - final Vulnerability vuln = await("Processed Vulnerability") - .atMost(Duration.ofSeconds(5)) - .until(() -> qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"), Objects::nonNull); + final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); assertThat(vuln.getVulnId()).isEqualTo("CVE-2022-40489"); assertThat(vuln.getFriendlyVulnId()).isNull(); From ed7f76261ee6018686d37c353dadcc8f461a5ac8 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 23:08:38 +0100 Subject: [PATCH 14/26] WIP: Handle more retryable exceptions Signed-off-by: nscuro --- .../api/AbstractRecordProcessingStrategy.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java index 5a9ab3b8c..0d1447c33 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java @@ -8,11 +8,14 @@ import org.datanucleus.api.jdo.exceptions.ConnectionInUseException; import org.datanucleus.store.query.QueryInterruptedException; import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; +import org.postgresql.util.PSQLState; import javax.jdo.JDOOptimisticVerificationException; import java.net.SocketTimeoutException; +import java.sql.SQLException; +import java.sql.SQLTransientConnectionException; import java.sql.SQLTransientException; -import java.util.concurrent.TimeoutException; +import java.util.List; /** * An abstract {@link RecordProcessingStrategy} that provides various shared functionality. @@ -54,19 +57,29 @@ ConsumerRecord deserialize(final ConsumerRecord record) { deserializedKey, deserializedValue, record.headers(), record.leaderEpoch()); } + private static final List> KNOWN_TRANSIENT_EXCEPTIONS = List.of( + ConnectTimeoutException.class, + ConnectionInUseException.class, + JDOOptimisticVerificationException.class, + QueryInterruptedException.class, + SocketTimeoutException.class, + SQLTransientException.class, + SQLTransientConnectionException.class + ); + boolean isRetryableException(final Throwable throwable) { if (throwable instanceof RetryableRecordProcessingException) { return true; } - final Throwable rootCause = ExceptionUtils.getRootCause(throwable); - return rootCause instanceof TimeoutException - || rootCause instanceof ConnectTimeoutException - || rootCause instanceof SocketTimeoutException - || rootCause instanceof ConnectionInUseException - || rootCause instanceof QueryInterruptedException - || rootCause instanceof JDOOptimisticVerificationException - || rootCause instanceof SQLTransientException; + final boolean isKnownTransientException = ExceptionUtils.getThrowableList(throwable).stream() + .anyMatch(cause -> KNOWN_TRANSIENT_EXCEPTIONS.contains(cause.getClass())); + if (isKnownTransientException) { + return true; + } + + return ExceptionUtils.getRootCause(throwable) instanceof final SQLException se + && PSQLState.isConnectionError(se.getSQLState()); } } From bf3934624217b5c3d88556cad6ef4ae88d80a164 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 28 Dec 2023 23:27:07 +0100 Subject: [PATCH 15/26] WIP: Fix missing `TimeoutException` check Signed-off-by: nscuro --- .../kafka/processor/api/AbstractRecordProcessingStrategy.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java index 0d1447c33..3de47153e 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java @@ -16,6 +16,7 @@ import java.sql.SQLTransientConnectionException; import java.sql.SQLTransientException; import java.util.List; +import java.util.concurrent.TimeoutException; /** * An abstract {@link RecordProcessingStrategy} that provides various shared functionality. @@ -64,7 +65,8 @@ ConsumerRecord deserialize(final ConsumerRecord record) { QueryInterruptedException.class, SocketTimeoutException.class, SQLTransientException.class, - SQLTransientConnectionException.class + SQLTransientConnectionException.class, + TimeoutException.class ); boolean isRetryableException(final Throwable throwable) { From 8dc71fd41d817db34cfb9f8ecdd664bfe9672b09 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 29 Dec 2023 21:10:13 +0100 Subject: [PATCH 16/26] Implement batch processor for processed vuln scan results Signed-off-by: nscuro --- ...essedVulnerabilityScanResultProcessor.java | 188 ++++++++++++++++++ .../mapping/VulnerabilityScanRowMapper.java | 31 +++ src/main/resources/application.properties | 5 + 3 files changed, 224 insertions(+) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java create mode 100644 src/main/java/org/dependencytrack/persistence/jdbi/mapping/VulnerabilityScanRowMapper.java diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java new file mode 100644 index 000000000..60a6b3add --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java @@ -0,0 +1,188 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.event.framework.ChainableEvent; +import alpine.event.framework.Event; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.event.ComponentMetricsUpdateEvent; +import org.dependencytrack.event.ComponentPolicyEvaluationEvent; +import org.dependencytrack.event.ProjectMetricsUpdateEvent; +import org.dependencytrack.event.ProjectPolicyEvaluationEvent; +import org.dependencytrack.event.kafka.processor.api.BatchRecordProcessor; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.model.VulnerabilityScan; +import org.dependencytrack.model.VulnerabilityScan.Status; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.jdbi.mapping.VulnerabilityScanRowMapper; +import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; +import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.statement.PreparedBatch; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import static java.lang.Math.toIntExact; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi; +import static org.dependencytrack.proto.vulnanalysis.v1.ScanStatus.SCAN_STATUS_FAILED; + +public class ProcessedVulnerabilityScanResultProcessor implements BatchRecordProcessor { + + // TODO: Find better name + public static final String PROCESSOR_NAME = "processed-vuln-scan-results"; + + @Override + public void process(final List> records) throws RecordProcessingException { + final List completedScans = processScanResults(records); + triggerPolicyEvalAndMetricsUpdates(completedScans); + } + + private static List processScanResults(final List> records) { + try (final var qm = new QueryManager()) { + return jdbi(qm).inTransaction(jdbiHandle -> { + final List completedScans = recordScanResults(jdbiHandle, records); + if (completedScans.isEmpty()) { + return completedScans; + } + + updateWorkflowSteps(jdbiHandle, completedScans); + + return completedScans; + }); + } + } + + private static List recordScanResults(final Handle jdbiHandle, final List> records) { + // As we may get multiple records for the same scan token, + // aggregate their respective values to reduce the number + // of SQL updates we have to execute. + final var aggregatesByToken = new HashMap(); + for (final ConsumerRecord record : records) { + aggregatesByToken.compute(record.key(), (token, existingAggregate) -> { + final ResultAggregate aggregate = Optional.ofNullable(existingAggregate) + .orElseGet(ResultAggregate::new); + aggregate.results++; + aggregate.scannerResultsTotal += record.value().getScannerResultsCount(); + aggregate.scannerResultsFailed += toIntExact(record.value().getScannerResultsList().stream() + .filter(result -> result.getStatus() == SCAN_STATUS_FAILED) + .count()); + return aggregate; + }); + } + + final PreparedBatch preparedBatch = jdbiHandle.prepareBatch(""" + UPDATE "VULNERABILITYSCAN" + SET + "RECEIVED_RESULTS" = "RECEIVED_RESULTS" + :results, + "SCAN_TOTAL" = "SCAN_TOTAL" + :scannerResultsTotal, + "SCAN_FAILED" = "SCAN_FAILED" + :scannerResultsFailed, + "STATUS" = ( + CASE + WHEN "EXPECTED_RESULTS" = ("RECEIVED_RESULTS" + :results) THEN + CASE + WHEN + (("SCAN_FAILED" + :scannerResultsFailed) + / ("SCAN_TOTAL" + :scannerResultsTotal)) > "FAILURE_THRESHOLD" + THEN + 'FAILED' + ELSE + 'COMPLETED' + END + ELSE 'IN_PROGRESS' + END + ), + "UPDATED_AT" = NOW() + WHERE + "TOKEN" = :token + RETURNING + "TOKEN", + "STATUS", + "TARGET_TYPE", + "TARGET_IDENTIFIER", + CASE + WHEN + "STATUS" = 'FAILED' + THEN + 'Failure threshold of ' || "FAILURE_THRESHOLD" || '% exceeded: ' + || "SCAN_FAILED" || '/' || "SCAN_TOTAL" || ' of scans failed' + END + """); + + for (final Map.Entry tokenAndAggregate : aggregatesByToken.entrySet()) { + final String token = tokenAndAggregate.getKey(); + final ResultAggregate aggregate = tokenAndAggregate.getValue(); + + preparedBatch + .bind("token", token) + .bind("results", aggregate.results) + .bind("scannerResultsTotal", aggregate.scannerResultsTotal) + .bind("scannerResultsFailed", aggregate.scannerResultsFailed) + .add(); + } + + return preparedBatch + .executePreparedBatch("TOKEN", "STATUS", "TARGET_TYPE", "TARGET_IDENTIFIER") + .map(new VulnerabilityScanRowMapper()) + .stream() + // Unfortunately we can't perform this filtering in SQL, as RETURNING + // does not allow a WHERE clause. Tried using a CTE as workaround: + // WITH "CTE" AS (UPDATE ... RETURNING ...) SELECT * FROM "CTE" + // but that didn't return any results at all. + .filter(vulnScan -> vulnScan.getStatus() == Status.COMPLETED + || vulnScan.getStatus() == Status.FAILED) + .toList(); + } + + private static void updateWorkflowSteps(final Handle jdbiHandle, final List completedScans) { + final PreparedBatch preparedBatch = jdbiHandle.prepareBatch(""" + UPDATE "WORKFLOW_STATE" + SET + "STATUS" = :status, + "FAILURE_REASON" = :failureReason, + "UPDATED_AT" = NOW() + WHERE + "TOKEN" = :token + """); + + for (final VulnerabilityScan vulnScan : completedScans) { + preparedBatch + .bind("token", vulnScan.getToken()) + .bind("status", vulnScan.getStatus()) + .bind("failureReason", vulnScan.getFailureReason()) + .add(); + } + + preparedBatch.executePreparedBatch(); + } + + private static void triggerPolicyEvalAndMetricsUpdates(final List completedScans) { + for (final VulnerabilityScan vulnScan : completedScans) { + final ChainableEvent policyEvaluationEvent = switch (vulnScan.getTargetType()) { + case COMPONENT -> new ComponentPolicyEvaluationEvent(vulnScan.getTargetIdentifier()); + case PROJECT -> new ProjectPolicyEvaluationEvent(vulnScan.getTargetIdentifier()); + }; + policyEvaluationEvent.setChainIdentifier(UUID.fromString(vulnScan.getToken())); + + final ChainableEvent metricsUpdateEvent = switch (vulnScan.getTargetType()) { + case COMPONENT -> new ComponentMetricsUpdateEvent(vulnScan.getTargetIdentifier()); + case PROJECT -> new ProjectMetricsUpdateEvent(vulnScan.getTargetIdentifier()); + }; + metricsUpdateEvent.setChainIdentifier(UUID.fromString(vulnScan.getToken())); + + policyEvaluationEvent.onFailure(metricsUpdateEvent); + policyEvaluationEvent.onSuccess(metricsUpdateEvent); + + Event.dispatch(policyEvaluationEvent); + } + } + + private static class ResultAggregate { + + private int results; + private int scannerResultsTotal; + private int scannerResultsFailed; + + } + +} diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/mapping/VulnerabilityScanRowMapper.java b/src/main/java/org/dependencytrack/persistence/jdbi/mapping/VulnerabilityScanRowMapper.java new file mode 100644 index 000000000..aaad46653 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/mapping/VulnerabilityScanRowMapper.java @@ -0,0 +1,31 @@ +package org.dependencytrack.persistence.jdbi.mapping; + +import org.dependencytrack.model.VulnerabilityScan; +import org.dependencytrack.model.VulnerabilityScan.Status; +import org.dependencytrack.model.VulnerabilityScan.TargetType; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.UUID; + +import static org.dependencytrack.persistence.jdbi.mapping.RowMapperUtil.maybeSet; + +public class VulnerabilityScanRowMapper implements RowMapper { + + @Override + public VulnerabilityScan map(final ResultSet rs, final StatementContext ctx) throws SQLException { + final var vulnScan = new VulnerabilityScan(); + maybeSet(rs, "token", ResultSet::getString, vulnScan::setToken); + maybeSet(rs, "scan_total", ResultSet::getLong, vulnScan::setScanTotal); + maybeSet(rs, "scan_failed", ResultSet::getLong, vulnScan::setScanFailed); + maybeSet(rs, "status", ResultSet::getString, status -> vulnScan.setStatus(Status.valueOf(status))); + maybeSet(rs, "target_type", ResultSet::getString, type -> vulnScan.setTargetType(TargetType.valueOf(type))); + maybeSet(rs, "target_identifier", ResultSet::getString, identifier -> vulnScan.setTargetIdentifier(UUID.fromString(identifier))); + maybeSet(rs, "failure_threshold", ResultSet::getDouble, vulnScan::setFailureThreshold); + maybeSet(rs, "failure_reason", ResultSet::getString, vulnScan::setFailureReason); + return vulnScan; + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 8ea6b180f..2130da43a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -453,6 +453,11 @@ alpine.kafka.processor.vuln-mirror.retry.multiplier=2 alpine.kafka.processor.vuln-mirror.retry.randomization.factor=0.3 alpine.kafka.processor.vuln-mirror.retry.max.delay.ms=180000 alpine.kafka.processor.vuln-mirror.consumer.group.id=dtrack-apiserver-processor +alpine.kafka.processor.processed-vuln-scan-results.processing.order=unordered +alpine.kafka.processor.processed-vuln-scan-results.max.batch.size=500 +alpine.kafka.processor.processed-vuln-scan-results.max.concurrency=1 +alpine.kafka.processor.processed-vuln-scan-results.consumer.group.id=dtrack-apiserver-processor +alpine.kafka.processor.processed-vuln-scan-results.consumer.max.poll.records=1000 # Scheduling tasks after 3 minutes (3*60*1000) of starting application task.scheduler.initial.delay=180000 From e125fd4f3e11aa9cf0fe28fbe1d33fd81f19733a Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 29 Dec 2023 21:41:55 +0100 Subject: [PATCH 17/26] Increase `fetch.min.bytes` for more effective batching Signed-off-by: nscuro --- src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 2130da43a..2f4a4a5b5 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -458,6 +458,7 @@ alpine.kafka.processor.processed-vuln-scan-results.max.batch.size=500 alpine.kafka.processor.processed-vuln-scan-results.max.concurrency=1 alpine.kafka.processor.processed-vuln-scan-results.consumer.group.id=dtrack-apiserver-processor alpine.kafka.processor.processed-vuln-scan-results.consumer.max.poll.records=1000 +alpine.kafka.processor.processed-vuln-scan-results.consumer.fetch.min.bytes=16384 # Scheduling tasks after 3 minutes (3*60*1000) of starting application task.scheduler.initial.delay=180000 From 40897cb774f0b64b3ec3ae57c79d5ccea5968488 Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 30 Dec 2023 01:15:56 +0100 Subject: [PATCH 18/26] Migrate `DelayedBomProcessedNotificationProcessor` Signed-off-by: nscuro --- .../kafka/KafkaDefaultProducerCallback.java | 10 +- .../event/kafka/KafkaEvent.java | 7 +- .../event/kafka/KafkaEventDispatcher.java | 57 +++++++- ...ayedBomProcessedNotificationProcessor.java | 125 ++++++++++++++++++ .../KafkaRecordProcessorInitializer.java | 6 + .../jdbi/NotificationSubjectDao.java | 40 +++++- .../mapping/NotificationBomRowMapper.java | 23 ++++ ...ubjectBomConsumedOrProcessedRowMapper.java | 32 +++++ src/main/resources/application.properties | 7 + 9 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java create mode 100644 src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationBomRowMapper.java create mode 100644 src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationSubjectBomConsumedOrProcessedRowMapper.java diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaDefaultProducerCallback.java b/src/main/java/org/dependencytrack/event/kafka/KafkaDefaultProducerCallback.java index efc4cefa6..95f35c715 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaDefaultProducerCallback.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaDefaultProducerCallback.java @@ -15,6 +15,10 @@ class KafkaDefaultProducerCallback implements Callback { private final String topic; private final Object key; + KafkaDefaultProducerCallback(final Logger logger) { + this(logger, null, null); + } + KafkaDefaultProducerCallback(final Logger logger, final String topic, final Object key) { this.logger = logger; this.topic = topic; @@ -27,7 +31,11 @@ class KafkaDefaultProducerCallback implements Callback { @Override public void onCompletion(final RecordMetadata metadata, final Exception exception) { if (exception != null) { - logger.error("Failed to produce record with key %s to topic %s".formatted(key, topic), exception); + if (topic != null) { + logger.error("Failed to produce record with key %s to topic %s".formatted(key, topic), exception); + } else { + logger.error("Failed to produce record", exception); + } } } diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEvent.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEvent.java index 8db04f3f3..b60db0bc3 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEvent.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEvent.java @@ -4,5 +4,10 @@ import java.util.Map; -record KafkaEvent(Topic topic, K key, V value, Map headers) { +public record KafkaEvent(Topic topic, K key, V value, Map headers) { + + public KafkaEvent(final Topic topic, final K key, final V value) { + this(topic, key, value, null); + } + } diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java index d5b94c0d9..564edaddf 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java @@ -18,9 +18,12 @@ import org.dependencytrack.model.Vulnerability; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -144,6 +147,50 @@ private Future dispatchAsyncInternal(final KafkaEvent new KafkaDefaultProducerCallback(LOGGER, event.topic().name(), event.key()))); + } + + public void dispatchAllBlocking(final List> events) { + dispatchAllBlocking(events, null); + } + + public void dispatchAllBlocking(final List> events, Callback callback) { + final var countDownLatch = new CountDownLatch(events.size()); + + callback = requireNonNullElseGet(callback, () -> new KafkaDefaultProducerCallback(LOGGER)); + callback = decorateCallback(callback, ((metadata, exception) -> countDownLatch.countDown())); + + dispatchAllAsync(events, callback); + + try { + countDownLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new KafkaException(""" + Thread was interrupted while waiting for all events to be acknowledged \ + by the broker. The acknowledgement of %d/%d events can not be determined.\ + """.formatted(countDownLatch.getCount(), events.size()), e); + } + } + + public List> dispatchAllAsync(final List> events, Callback callback) { + final var records = new ArrayList>(events.size()); + for (final KafkaEvent event : events) { + records.add(toProducerRecord(event)); + } + + callback = requireNonNullElseGet(callback, () -> new KafkaDefaultProducerCallback(LOGGER)); + + final var futures = new ArrayList>(records.size()); + for (final ProducerRecord record : records) { + futures.add(producer.send(record, callback)); + } + + return futures; + } + + private static ProducerRecord toProducerRecord(final KafkaEvent event) { final byte[] keyBytes; try (final Serde keySerde = event.topic().keySerde()) { keyBytes = keySerde.serializer().serialize(event.topic().name(), event.key()); @@ -165,8 +212,14 @@ final var record = new ProducerRecord<>(event.topic().name(), keyBytes, valueByt } } - return producer.send(record, requireNonNullElseGet(callback, - () -> new KafkaDefaultProducerCallback(LOGGER, record.topic(), event.key()))); + return record; + } + + private static Callback decorateCallback(final Callback originalCallback, final Callback decoratorCallback) { + return (metadata, exception) -> { + decoratorCallback.onCompletion(metadata, exception); + originalCallback.onCompletion(metadata, exception); + }; } } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java new file mode 100644 index 000000000..d222a2587 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java @@ -0,0 +1,125 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.Config; +import alpine.common.logging.Logger; +import com.google.protobuf.Any; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.kafka.KafkaEvent; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.event.kafka.processor.api.BatchRecordProcessor; +import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.model.VulnerabilityScan; +import org.dependencytrack.notification.NotificationConstants; +import org.dependencytrack.notification.NotificationGroup; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.jdbi.NotificationSubjectDao; +import org.dependencytrack.proto.notification.v1.BomConsumedOrProcessedSubject; +import org.dependencytrack.proto.notification.v1.Notification; +import org.dependencytrack.proto.notification.v1.ProjectVulnAnalysisCompleteSubject; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi; +import static org.dependencytrack.proto.notification.v1.Group.GROUP_BOM_PROCESSED; +import static org.dependencytrack.proto.notification.v1.Level.LEVEL_INFORMATIONAL; +import static org.dependencytrack.proto.notification.v1.Scope.SCOPE_PORTFOLIO; + +/** + * A {@link BatchRecordProcessor} responsible for dispatching {@link NotificationGroup#BOM_PROCESSED} notifications + * upon detection of a completed {@link VulnerabilityScan}. + *

+ * The completion detection is based on {@link NotificationGroup#PROJECT_VULN_ANALYSIS_COMPLETE} notifications. + * This processor does nothing unless {@link ConfigKey#TMP_DELAY_BOM_PROCESSED_NOTIFICATION} is enabled. + */ +public class DelayedBomProcessedNotificationProcessor implements BatchRecordProcessor { + + // TODO: Find better name + public static final String PROCESSOR_NAME = "delayed-bom-processed-notification"; + + private static final Logger LOGGER = Logger.getLogger(DelayedBomProcessedNotificationProcessor.class); + + private final Config config; + private final KafkaEventDispatcher eventDispatcher; + + public DelayedBomProcessedNotificationProcessor() { + this(Config.getInstance(), new KafkaEventDispatcher()); + } + + DelayedBomProcessedNotificationProcessor(final Config config, final KafkaEventDispatcher eventDispatcher) { + this.config = config; + this.eventDispatcher = eventDispatcher; + } + + @Override + public void process(final List> records) throws RecordProcessingException { + if (!config.getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION)) { + return; + } + + final Set tokens = extractTokens(records); + if (tokens.isEmpty()) { + LOGGER.warn("No token could be extracted from any of the %d records in this batch" + .formatted(records.size())); + return; + } + + final List subjects; + try (final var qm = new QueryManager()) { + subjects = jdbi(qm).withExtension(NotificationSubjectDao.class, + dao -> dao.getForDelayedBomProcessed(tokens)); + } + + dispatchNotifications(subjects); + } + + private static Set extractTokens(final List> records) { + final var tokens = new HashSet(); + for (final ConsumerRecord record : records) { + final Notification notification = record.value(); + if (!notification.hasSubject() || !notification.getSubject().is(ProjectVulnAnalysisCompleteSubject.class)) { + continue; + } + + final ProjectVulnAnalysisCompleteSubject subject; + try { + subject = notification.getSubject().unpack(ProjectVulnAnalysisCompleteSubject.class); + } catch (InvalidProtocolBufferException e) { + LOGGER.warn("Failed to unpack notification subject from %s; Skipping".formatted(record), e); + continue; + } + + tokens.add(subject.getToken()); + } + + return tokens; + } + + private void dispatchNotifications(final List subjects) { + final Timestamp timestamp = Timestamps.now(); + final var events = new ArrayList>(subjects.size()); + for (final BomConsumedOrProcessedSubject subject : subjects) { + final var event = new KafkaEvent<>(KafkaTopics.NOTIFICATION_BOM, + subject.getProject().getUuid(), Notification.newBuilder() + .setScope(SCOPE_PORTFOLIO) + .setGroup(GROUP_BOM_PROCESSED) + .setLevel(LEVEL_INFORMATIONAL) + .setTimestamp(timestamp) + .setTitle(NotificationConstants.Title.BOM_PROCESSED) + .setContent("A %s BOM was processed".formatted(subject.getBom().getFormat())) + .setSubject(Any.pack(subject)) + .build()); + events.add(event); + } + + eventDispatcher.dispatchAllBlocking(events); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java index e93314008..546725540 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java @@ -1,6 +1,8 @@ package org.dependencytrack.event.kafka.processor; +import alpine.Config; import alpine.common.logging.Logger; +import org.dependencytrack.common.ConfigKey; import org.dependencytrack.event.kafka.KafkaTopics; import org.dependencytrack.event.kafka.processor.api.RecordProcessorManager; @@ -21,6 +23,10 @@ public void contextInitialized(final ServletContextEvent event) { new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); processorManager.register(RepositoryMetaResultProcessor.PROCESSOR_NAME, new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); + if (Config.getInstance().getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION)) { + processorManager.register(DelayedBomProcessedNotificationProcessor.PROCESSOR_NAME, + new DelayedBomProcessedNotificationProcessor(), KafkaTopics.NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE); + } processorManager.startAll(); } diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/NotificationSubjectDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/NotificationSubjectDao.java index f87bb7e6c..824113ac6 100644 --- a/src/main/java/org/dependencytrack/persistence/jdbi/NotificationSubjectDao.java +++ b/src/main/java/org/dependencytrack/persistence/jdbi/NotificationSubjectDao.java @@ -1,16 +1,20 @@ package org.dependencytrack.persistence.jdbi; import org.dependencytrack.model.VulnerabilityAnalysisLevel; +import org.dependencytrack.persistence.jdbi.mapping.NotificationBomRowMapper; import org.dependencytrack.persistence.jdbi.mapping.NotificationComponentRowMapper; import org.dependencytrack.persistence.jdbi.mapping.NotificationProjectRowMapper; +import org.dependencytrack.persistence.jdbi.mapping.NotificationSubjectBomConsumedOrProcessedRowMapper; import org.dependencytrack.persistence.jdbi.mapping.NotificationSubjectNewVulnerabilityRowMapper; import org.dependencytrack.persistence.jdbi.mapping.NotificationSubjectNewVulnerableDependencyRowReducer; import org.dependencytrack.persistence.jdbi.mapping.NotificationVulnerabilityRowMapper; +import org.dependencytrack.proto.notification.v1.BomConsumedOrProcessedSubject; import org.dependencytrack.proto.notification.v1.NewVulnerabilitySubject; import org.dependencytrack.proto.notification.v1.NewVulnerableDependencySubject; import org.jdbi.v3.sqlobject.config.RegisterRowMapper; import org.jdbi.v3.sqlobject.config.RegisterRowMappers; import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.UseRowMapper; import org.jdbi.v3.sqlobject.statement.UseRowReducer; import java.util.Collection; @@ -19,6 +23,7 @@ import java.util.UUID; @RegisterRowMappers({ + @RegisterRowMapper(NotificationBomRowMapper.class), @RegisterRowMapper(NotificationComponentRowMapper.class), @RegisterRowMapper(NotificationProjectRowMapper.class), @RegisterRowMapper(NotificationVulnerabilityRowMapper.class) @@ -128,7 +133,7 @@ LEFT JOIN LATERAL ( "C"."UUID" = (:componentUuid)::TEXT AND "V"."UUID" = ANY((:vulnUuids)::TEXT[]) AND ("A"."SUPPRESSED" IS NULL OR NOT "A"."SUPPRESSED") """) - @RegisterRowMapper(NotificationSubjectNewVulnerabilityRowMapper.class) + @UseRowMapper(NotificationSubjectNewVulnerabilityRowMapper.class) List getForNewVulnerabilities(final UUID componentUuid, final Collection vulnUuids, final VulnerabilityAnalysisLevel vulnAnalysisLevel); @@ -235,4 +240,37 @@ LEFT JOIN LATERAL ( @UseRowReducer(NotificationSubjectNewVulnerableDependencyRowReducer.class) Optional getForNewVulnerableDependency(final UUID componentUuid); + @SqlQuery(""" + SELECT + "P"."UUID" AS "projectUuid", + "P"."NAME" AS "projectName", + "P"."VERSION" AS "projectVersion", + "P"."DESCRIPTION" AS "projectDescription", + "P"."PURL" AS "projectPurl", + (SELECT + ARRAY_AGG(DISTINCT "T"."NAME") + FROM + "TAG" AS "T" + INNER JOIN + "PROJECTS_TAGS" AS "PT" ON "PT"."TAG_ID" = "T"."ID" + WHERE + "PT"."PROJECT_ID" = "P"."ID" + ) AS "projectTags", + 'CycloneDX' AS "bomFormat", + '(Unknown)' AS "bomSpecVersion", + '(Omitted)' AS "bomContent" + FROM + "VULNERABILITYSCAN" AS "VS" + INNER JOIN + "PROJECT" AS "P" ON "P"."UUID" = "VS"."TARGET_IDENTIFIER" + INNER JOIN + "WORKFLOW_STATE" AS "WFS" ON "WFS"."TOKEN" = "VS"."TOKEN" + AND "WFS"."STEP" = 'BOM_PROCESSING' + AND "WFS"."STATUS" = 'COMPLETED' + WHERE + "VS"."TOKEN" = ANY(:tokens) + """) + @UseRowMapper(NotificationSubjectBomConsumedOrProcessedRowMapper.class) + List getForDelayedBomProcessed(final Collection tokens); + } diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationBomRowMapper.java b/src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationBomRowMapper.java new file mode 100644 index 000000000..d6e53e68f --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationBomRowMapper.java @@ -0,0 +1,23 @@ +package org.dependencytrack.persistence.jdbi.mapping; + +import org.dependencytrack.proto.notification.v1.Bom; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.dependencytrack.persistence.jdbi.mapping.RowMapperUtil.maybeSet; + +public class NotificationBomRowMapper implements RowMapper { + + @Override + public Bom map(final ResultSet rs, final StatementContext ctx) throws SQLException { + final Bom.Builder builder = Bom.newBuilder(); + maybeSet(rs, "bomFormat", ResultSet::getString, builder::setFormat); + maybeSet(rs, "bomSpecVersion", ResultSet::getString, builder::setSpecVersion); + maybeSet(rs, "bomContent", ResultSet::getString, builder::setContent); + return builder.build(); + } + +} diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationSubjectBomConsumedOrProcessedRowMapper.java b/src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationSubjectBomConsumedOrProcessedRowMapper.java new file mode 100644 index 000000000..66b1c4036 --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/mapping/NotificationSubjectBomConsumedOrProcessedRowMapper.java @@ -0,0 +1,32 @@ +package org.dependencytrack.persistence.jdbi.mapping; + +import org.dependencytrack.proto.notification.v1.Bom; +import org.dependencytrack.proto.notification.v1.BomConsumedOrProcessedSubject; +import org.dependencytrack.proto.notification.v1.Project; +import org.jdbi.v3.core.mapper.NoSuchMapperException; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.dependencytrack.persistence.jdbi.mapping.RowMapperUtil.maybeSet; + +public class NotificationSubjectBomConsumedOrProcessedRowMapper implements RowMapper { + + @Override + public BomConsumedOrProcessedSubject map(final ResultSet rs, final StatementContext ctx) throws SQLException { + final RowMapper projectRowMapper = ctx.findRowMapperFor(Project.class) + .orElseThrow(() -> new NoSuchMapperException("No mapper registered for %s".formatted(Project.class))); + final RowMapper bomRowMapper = ctx.findRowMapperFor(Bom.class) + .orElseThrow(() -> new NoSuchMapperException("No mapper registered for %s".formatted(Bom.class))); + + final BomConsumedOrProcessedSubject.Builder builder = BomConsumedOrProcessedSubject.newBuilder() + .setProject(projectRowMapper.map(rs, ctx)) + .setBom(bomRowMapper.map(rs, ctx)); + maybeSet(rs, "token", ResultSet::getString, builder::setToken); + + return builder.build(); + } + +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 2f4a4a5b5..798f07b2e 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -446,6 +446,7 @@ kafka.streams.transient.processing.exception.threshold.interval=PT30M alpine.kafka.processor.repo-meta-result.max.concurrency=3 alpine.kafka.processor.repo-meta-result.processing.order=partition alpine.kafka.processor.repo-meta-result.consumer.group.id=dtrack-apiserver-processor + alpine.kafka.processor.vuln-mirror.max.concurrency=3 alpine.kafka.processor.vuln-mirror.processing.order=partition alpine.kafka.processor.vuln-mirror.retry.initial.delay.ms=3000 @@ -453,6 +454,7 @@ alpine.kafka.processor.vuln-mirror.retry.multiplier=2 alpine.kafka.processor.vuln-mirror.retry.randomization.factor=0.3 alpine.kafka.processor.vuln-mirror.retry.max.delay.ms=180000 alpine.kafka.processor.vuln-mirror.consumer.group.id=dtrack-apiserver-processor + alpine.kafka.processor.processed-vuln-scan-results.processing.order=unordered alpine.kafka.processor.processed-vuln-scan-results.max.batch.size=500 alpine.kafka.processor.processed-vuln-scan-results.max.concurrency=1 @@ -460,6 +462,11 @@ alpine.kafka.processor.processed-vuln-scan-results.consumer.group.id=dtrack-apis alpine.kafka.processor.processed-vuln-scan-results.consumer.max.poll.records=1000 alpine.kafka.processor.processed-vuln-scan-results.consumer.fetch.min.bytes=16384 +alpine.kafka.processor.delayed-bom-processed-notification.processing.order=unordered +alpine.kafka.processor.delayed-bom-processed-notification.max.batch.size=100 +alpine.kafka.processor.delayed-bom-processed-notification.max.concurrency=1 +alpine.kafka.processor.delayed-bom-processed-notification.consumer.group.id=dtrack-apiserver-processor + # Scheduling tasks after 3 minutes (3*60*1000) of starting application task.scheduler.initial.delay=180000 From befe8cbd5d624604d6cb1703dd3e9c0685d8702e Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 30 Dec 2023 02:30:22 +0100 Subject: [PATCH 19/26] Add health check for Kafka processors Signed-off-by: nscuro --- .../processor/KafkaProcessorsHealthCheck.java | 17 ++++++++++++ ...r.java => KafkaProcessorsInitializer.java} | 17 ++++++------ .../processor/api/RecordProcessorManager.java | 27 +++++++++++++++++++ .../health/HealthCheckInitializer.java | 3 ++- src/main/webapp/WEB-INF/web.xml | 2 +- 5 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsHealthCheck.java rename src/main/java/org/dependencytrack/event/kafka/processor/{KafkaRecordProcessorInitializer.java => KafkaProcessorsInitializer.java} (67%) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsHealthCheck.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsHealthCheck.java new file mode 100644 index 000000000..d70862b18 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsHealthCheck.java @@ -0,0 +1,17 @@ +package org.dependencytrack.event.kafka.processor; + +import org.eclipse.microprofile.health.HealthCheck; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.eclipse.microprofile.health.Liveness; + +import static org.dependencytrack.event.kafka.processor.KafkaProcessorsInitializer.PROCESSOR_MANAGER; + +@Liveness +public class KafkaProcessorsHealthCheck implements HealthCheck { + + @Override + public HealthCheckResponse call() { + return PROCESSOR_MANAGER.probeHealth(); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java similarity index 67% rename from src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java rename to src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java index 546725540..8f23379ff 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaRecordProcessorInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java @@ -9,31 +9,32 @@ import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; -public class KafkaRecordProcessorInitializer implements ServletContextListener { +public class KafkaProcessorsInitializer implements ServletContextListener { - private static final Logger LOGGER = Logger.getLogger(KafkaRecordProcessorInitializer.class); + private static final Logger LOGGER = Logger.getLogger(KafkaProcessorsInitializer.class); - private final RecordProcessorManager processorManager = new RecordProcessorManager(); + static final RecordProcessorManager PROCESSOR_MANAGER = new RecordProcessorManager(); @Override public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Initializing Kafka processors"); - processorManager.register(VulnerabilityMirrorProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.register(VulnerabilityMirrorProcessor.PROCESSOR_NAME, new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); - processorManager.register(RepositoryMetaResultProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.register(RepositoryMetaResultProcessor.PROCESSOR_NAME, new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); if (Config.getInstance().getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION)) { - processorManager.register(DelayedBomProcessedNotificationProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.register(DelayedBomProcessedNotificationProcessor.PROCESSOR_NAME, new DelayedBomProcessedNotificationProcessor(), KafkaTopics.NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE); } - processorManager.startAll(); + + PROCESSOR_MANAGER.startAll(); } @Override public void contextDestroyed(final ServletContextEvent event) { LOGGER.info("Stopping Kafka processors"); - processorManager.close(); + PROCESSOR_MANAGER.close(); } } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java index 317084dfb..9c9139674 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java @@ -5,6 +5,7 @@ import alpine.common.metrics.Metrics; import io.confluent.parallelconsumer.ParallelConsumerOptions; import io.confluent.parallelconsumer.ParallelConsumerOptions.ProcessingOrder; +import io.confluent.parallelconsumer.ParallelEoSStreamProcessor; import io.confluent.parallelconsumer.ParallelStreamProcessor; import io.github.resilience4j.core.IntervalFunction; import io.micrometer.core.instrument.binder.kafka.KafkaClientMetrics; @@ -14,6 +15,7 @@ import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.dependencytrack.event.kafka.KafkaTopics.Topic; +import org.eclipse.microprofile.health.HealthCheckResponse; import java.time.Duration; import java.util.HashMap; @@ -87,6 +89,31 @@ public void startAll() { } } + public HealthCheckResponse probeHealth() { + final var responseBuilder = HealthCheckResponse.named("kafka-processors"); + + boolean isUp = true; + for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { + final String processorName = managedProcessorEntry.getKey(); + final ParallelStreamProcessor parallelConsumer = managedProcessorEntry.getValue().parallelConsumer(); + final boolean isProcessorUp = !parallelConsumer.isClosedOrFailed(); + + responseBuilder.withData(processorName, isProcessorUp + ? HealthCheckResponse.Status.UP.name() + : HealthCheckResponse.Status.DOWN.name()); + if (isProcessorUp + && parallelConsumer instanceof final ParallelEoSStreamProcessor concreteParallelConsumer + && concreteParallelConsumer.getFailureCause() != null) { + responseBuilder.withData("%s_failure_reason".formatted(processorName), + concreteParallelConsumer.getFailureCause().getMessage()); + } + + isUp &= isProcessorUp; + } + + return responseBuilder.status(isUp).build(); + } + @Override public void close() { for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { diff --git a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java index 60ad5dd04..7b7e9112f 100644 --- a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java +++ b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java @@ -24,6 +24,7 @@ import alpine.server.health.checks.DatabaseHealthCheck; import io.github.mweirauch.micrometer.jvm.extras.ProcessMemoryMetrics; import io.github.mweirauch.micrometer.jvm.extras.ProcessThreadMetrics; +import org.dependencytrack.event.kafka.processor.KafkaProcessorsHealthCheck; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -36,7 +37,7 @@ public class HealthCheckInitializer implements ServletContextListener { public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Registering health checks"); HealthCheckRegistry.getInstance().register("database", new DatabaseHealthCheck()); - HealthCheckRegistry.getInstance().register("kafka-streams", new KafkaStreamsHealthCheck()); + HealthCheckRegistry.getInstance().register("kafka-processors", new KafkaProcessorsHealthCheck()); // TODO: Move this to its own initializer if it turns out to be useful LOGGER.info("Registering extra process metrics"); diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index d6b206117..099a11dc1 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -48,7 +48,7 @@ org.dependencytrack.event.EventSubsystemInitializer - org.dependencytrack.event.kafka.processor.KafkaRecordProcessorInitializer + org.dependencytrack.event.kafka.processor.KafkaProcessorsInitializer org.dependencytrack.event.kafka.streams.KafkaStreamsInitializer From 7e2a1237662ef9203983b553c95b13b93ec019ba Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 30 Dec 2023 18:12:50 +0100 Subject: [PATCH 20/26] Ensure processor properties can be provided via env vars Signed-off-by: nscuro --- .../event/kafka/KafkaTopics.java | 2 + ...ayedBomProcessedNotificationProcessor.java | 3 +- .../processor/KafkaProcessorsInitializer.java | 8 +- ...essedVulnerabilityScanResultProcessor.java | 3 +- .../RepositoryMetaResultProcessor.java | 2 +- .../VulnerabilityMirrorProcessor.java | 2 +- .../processor/api/RecordProcessorManager.java | 120 ++++++++++++------ .../health/HealthCheckInitializer.java | 4 +- src/main/resources/application.properties | 48 +++---- .../api/RecordProcessorManagerTest.java | 6 +- 10 files changed, 119 insertions(+), 79 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaTopics.java b/src/main/java/org/dependencytrack/event/kafka/KafkaTopics.java index fc21ed444..1dd164e5b 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaTopics.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaTopics.java @@ -34,6 +34,7 @@ public final class KafkaTopics { public static final Topic REPO_META_ANALYSIS_RESULT; public static final Topic VULN_ANALYSIS_COMMAND; public static final Topic VULN_ANALYSIS_RESULT; + public static final Topic VULN_ANALYSIS_RESULT_PROCESSED; public static final Topic NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE; private static final Serde NOTIFICATION_SERDE = new KafkaProtobufSerde<>(Notification.parser()); @@ -58,6 +59,7 @@ public final class KafkaTopics { REPO_META_ANALYSIS_RESULT = new Topic<>("dtrack.repo-meta-analysis.result", Serdes.String(), new KafkaProtobufSerde<>(AnalysisResult.parser())); VULN_ANALYSIS_COMMAND = new Topic<>("dtrack.vuln-analysis.component", new KafkaProtobufSerde<>(ScanKey.parser()), new KafkaProtobufSerde<>(ScanCommand.parser())); VULN_ANALYSIS_RESULT = new Topic<>("dtrack.vuln-analysis.result", new KafkaProtobufSerde<>(ScanKey.parser()), new KafkaProtobufSerde<>(ScanResult.parser())); + VULN_ANALYSIS_RESULT_PROCESSED = new Topic<>("dtrack.vuln-analysis.result.processed", Serdes.String(), new KafkaProtobufSerde<>(ScanResult.parser())); NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE = new Topic<>("dtrack.notification.project-vuln-analysis-complete", Serdes.String(), NOTIFICATION_SERDE); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java index d222a2587..13828a9d2 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java @@ -41,8 +41,7 @@ */ public class DelayedBomProcessedNotificationProcessor implements BatchRecordProcessor { - // TODO: Find better name - public static final String PROCESSOR_NAME = "delayed-bom-processed-notification"; + public static final String PROCESSOR_NAME = "delayed.bom.processed.notification"; private static final Logger LOGGER = Logger.getLogger(DelayedBomProcessedNotificationProcessor.class); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java index 8f23379ff..fd558e4aa 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java @@ -19,12 +19,14 @@ public class KafkaProcessorsInitializer implements ServletContextListener { public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Initializing Kafka processors"); - PROCESSOR_MANAGER.register(VulnerabilityMirrorProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.registerProcessor(VulnerabilityMirrorProcessor.PROCESSOR_NAME, new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); - PROCESSOR_MANAGER.register(RepositoryMetaResultProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.registerProcessor(RepositoryMetaResultProcessor.PROCESSOR_NAME, new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); + PROCESSOR_MANAGER.registerBatchProcessor(ProcessedVulnerabilityScanResultProcessor.PROCESSOR_NAME, + new ProcessedVulnerabilityScanResultProcessor(), KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED); if (Config.getInstance().getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION)) { - PROCESSOR_MANAGER.register(DelayedBomProcessedNotificationProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.registerBatchProcessor(DelayedBomProcessedNotificationProcessor.PROCESSOR_NAME, new DelayedBomProcessedNotificationProcessor(), KafkaTopics.NOTIFICATION_PROJECT_VULN_ANALYSIS_COMPLETE); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java index 60a6b3add..533b44c61 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java @@ -29,8 +29,7 @@ public class ProcessedVulnerabilityScanResultProcessor implements BatchRecordProcessor { - // TODO: Find better name - public static final String PROCESSOR_NAME = "processed-vuln-scan-results"; + public static final String PROCESSOR_NAME = "processed.vuln.scan.result"; @Override public void process(final List> records) throws RecordProcessingException { diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index bb6d716d9..6c67b723b 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -32,7 +32,7 @@ */ public class RepositoryMetaResultProcessor implements SingleRecordProcessor { - public static final String PROCESSOR_NAME = "repo-meta-result"; + public static final String PROCESSOR_NAME = "repo.meta.result"; private static final Logger LOGGER = Logger.getLogger(RepositoryMetaResultProcessor.class); private static final Timer TIMER = Timer.builder("repo_meta_result_processing") diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java index 786300728..2c5bb078d 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java @@ -33,7 +33,7 @@ */ public class VulnerabilityMirrorProcessor implements SingleRecordProcessor { - public static final String PROCESSOR_NAME = "vuln-mirror"; + public static final String PROCESSOR_NAME = "mirrored.vuln"; private static final Logger LOGGER = Logger.getLogger(VulnerabilityMirrorProcessor.class); private static final Timer TIMER = Timer.builder("vuln_mirror_processing") diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java index 9c9139674..b170736b2 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java @@ -19,6 +19,7 @@ import java.time.Duration; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -36,23 +37,24 @@ public class RecordProcessorManager implements AutoCloseable { private static final Logger LOGGER = Logger.getLogger(RecordProcessorManager.class); + private static final Pattern PROCESSOR_NAME_PATTERN = Pattern.compile("^[a-z.]+$"); private static final String PROPERTY_MAX_BATCH_SIZE = "max.batch.size"; private static final int PROPERTY_MAX_BATCH_SIZE_DEFAULT = 10; private static final String PROPERTY_MAX_CONCURRENCY = "max.concurrency"; - private static final int PROPERTY_MAX_CONCURRENCY_DEFAULT = 3; + private static final int PROPERTY_MAX_CONCURRENCY_DEFAULT = 1; private static final String PROPERTY_PROCESSING_ORDER = "processing.order"; private static final ProcessingOrder PROPERTY_PROCESSING_ORDER_DEFAULT = ProcessingOrder.PARTITION; private static final String PROPERTY_RETRY_INITIAL_DELAY_MS = "retry.initial.delay.ms"; - private static final long PROPERTY_RETRY_INITIAL_DELAY_MS_DEFAULT = 1000; + private static final long PROPERTY_RETRY_INITIAL_DELAY_MS_DEFAULT = 1000; // 1s private static final String PROPERTY_RETRY_MULTIPLIER = "retry.multiplier"; private static final int PROPERTY_RETRY_MULTIPLIER_DEFAULT = 1; private static final String PROPERTY_RETRY_RANDOMIZATION_FACTOR = "retry.randomization.factor"; private static final double PROPERTY_RETRY_RANDOMIZATION_FACTOR_DEFAULT = 0.3; private static final String PROPERTY_RETRY_MAX_DELAY_MS = "retry.max.delay.ms"; - private static final long PROPERTY_RETRY_MAX_DELAY_MS_DEFAULT = 60 * 1000; + private static final long PROPERTY_RETRY_MAX_DELAY_MS_DEFAULT = 60 * 1000; // 60s - private final Map managedProcessors = new LinkedHashMap<>(); + private final Map managedProcessors = new LinkedHashMap<>(); private final Config config; public RecordProcessorManager() { @@ -63,25 +65,46 @@ public RecordProcessorManager(final Config config) { this.config = config; } - public void register(final String name, final SingleRecordProcessor recordProcessor, final Topic topic) { - final var processingStrategy = new SingleRecordProcessingStrategy<>(recordProcessor, topic.keySerde(), topic.valueSerde()); - final var parallelConsumer = createParallelConsumer(name, false); - managedProcessors.put(name, new ManagedRecordProcessor(parallelConsumer, processingStrategy, topic)); + /** + * Register a new {@link SingleRecordProcessor}. + * + * @param name Name of the processor to register + * @param processor The processor to register + * @param topic The topic to have the processor subscribe to + * @param Type of record keys in the topic + * @param Type of record values in the topic + */ + public void registerProcessor(final String name, final SingleRecordProcessor processor, final Topic topic) { + requireValidProcessorName(name); + final var processingStrategy = new SingleRecordProcessingStrategy<>(processor, topic.keySerde(), topic.valueSerde()); + final ParallelStreamProcessor parallelConsumer = createParallelConsumer(name, false); + managedProcessors.put(name, new ManagedProcessor(parallelConsumer, processingStrategy, topic.name())); } - public void register(final String name, final BatchRecordProcessor recordProcessor, final Topic topic) { - final var processingStrategy = new BatchRecordProcessingStrategy<>(recordProcessor, topic.keySerde(), topic.valueSerde()); - final var parallelConsumer = createParallelConsumer(name, true); - managedProcessors.put(name, new ManagedRecordProcessor(parallelConsumer, processingStrategy, topic)); + /** + * Register a new {@link BatchRecordProcessor}. + * + * @param name Name of the processor to register + * @param processor The processor to register + * @param topic The topic to have the processor subscribe to + * @param Type of record keys in the topic + * @param Type of record values in the topic + */ + public void registerBatchProcessor(final String name, final BatchRecordProcessor processor, final Topic topic) { + requireValidProcessorName(name); + final var processingStrategy = new BatchRecordProcessingStrategy<>(processor, topic.keySerde(), topic.valueSerde()); + final ParallelStreamProcessor parallelConsumer = createParallelConsumer(name, true); + managedProcessors.put(name, new ManagedProcessor(parallelConsumer, processingStrategy, topic.name())); } + @SuppressWarnings("resource") public void startAll() { - for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { - final String processorName = managedProcessorEntry.getKey(); - final ManagedRecordProcessor managedProcessor = managedProcessorEntry.getValue(); + for (final Map.Entry entry : managedProcessors.entrySet()) { + final String processorName = entry.getKey(); + final ManagedProcessor managedProcessor = entry.getValue(); LOGGER.info("Starting processor %s".formatted(processorName)); - managedProcessor.parallelConsumer().subscribe(List.of(managedProcessor.topic().name())); + managedProcessor.parallelConsumer().subscribe(List.of(managedProcessor.topic())); managedProcessor.parallelConsumer().poll(pollCtx -> { final List> polledRecords = pollCtx.getConsumerRecordsFlattened(); managedProcessor.processingStrategy().processRecords(polledRecords); @@ -93,9 +116,9 @@ public HealthCheckResponse probeHealth() { final var responseBuilder = HealthCheckResponse.named("kafka-processors"); boolean isUp = true; - for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { - final String processorName = managedProcessorEntry.getKey(); - final ParallelStreamProcessor parallelConsumer = managedProcessorEntry.getValue().parallelConsumer(); + for (final Map.Entry entry : managedProcessors.entrySet()) { + final String processorName = entry.getKey(); + final ParallelStreamProcessor parallelConsumer = entry.getValue().parallelConsumer(); final boolean isProcessorUp = !parallelConsumer.isClosedOrFailed(); responseBuilder.withData(processorName, isProcessorUp @@ -115,13 +138,17 @@ public HealthCheckResponse probeHealth() { } @Override + @SuppressWarnings("resource") public void close() { - for (final Map.Entry managedProcessorEntry : managedProcessors.entrySet()) { - final String processorName = managedProcessorEntry.getKey(); - final ManagedRecordProcessor managedProcessor = managedProcessorEntry.getValue(); + final Iterator> entryIterator = managedProcessors.entrySet().iterator(); + while (entryIterator.hasNext()) { + final Map.Entry entry = entryIterator.next(); + final String processorName = entry.getKey(); + final ManagedProcessor managedProcessor = entry.getValue(); LOGGER.info("Stopping processor %s".formatted(processorName)); - managedProcessor.parallelConsumer().closeDrainFirst(); + managedProcessor.parallelConsumer().closeDontDrainFirst(); + entryIterator.remove(); } } @@ -129,21 +156,12 @@ private ParallelStreamProcessor createParallelConsumer(final Str final var optionsBuilder = ParallelConsumerOptions.builder() .consumer(createConsumer(processorName)); - final Map properties = getPassThroughProperties(processorName); + final Map properties = getPassThroughProperties(processorName.toLowerCase()); final ProcessingOrder processingOrder = Optional.ofNullable(properties.get(PROPERTY_PROCESSING_ORDER)) .map(String::toUpperCase) .map(ProcessingOrder::valueOf) .orElse(PROPERTY_PROCESSING_ORDER_DEFAULT); - if (processingOrder != ProcessingOrder.PARTITION) { - LOGGER.warn(""" - Processor %s is configured to use ordering mode %s; \ - Ordering modes other than %s bypass head-of-line blocking \ - and can put additional strain on the system in cases where \ - external systems like the database are temporarily unavailable \ - (https://github.com/confluentinc/parallel-consumer#head-of-line-blocking)\ - """.formatted(processorName, processingOrder, ProcessingOrder.PARTITION)); - } optionsBuilder.ordering(processingOrder); final int maxConcurrency = Optional.ofNullable(properties.get(PROPERTY_MAX_CONCURRENCY)) @@ -151,6 +169,7 @@ private ParallelStreamProcessor createParallelConsumer(final Str .orElse(PROPERTY_MAX_CONCURRENCY_DEFAULT); optionsBuilder.maxConcurrency(maxConcurrency); + final Optional optionalMaxBatchSizeProperty = Optional.ofNullable(properties.get(PROPERTY_MAX_BATCH_SIZE)); if (isBatch) { if (processingOrder == ProcessingOrder.PARTITION) { LOGGER.warn(""" @@ -161,10 +180,13 @@ private ParallelStreamProcessor createParallelConsumer(final Str """.formatted(processorName, processingOrder)); } - final int maxBatchSize = Optional.ofNullable(properties.get(PROPERTY_MAX_BATCH_SIZE)) + final int maxBatchSize = optionalMaxBatchSizeProperty .map(Integer::parseInt) .orElse(PROPERTY_MAX_BATCH_SIZE_DEFAULT); optionsBuilder.batchSize(maxBatchSize); + } else if (optionalMaxBatchSizeProperty.isPresent()) { + LOGGER.warn("Processor %s is configured with %s, but it is not a batch processor; Ignoring property" + .formatted(processorName, PROPERTY_MAX_BATCH_SIZE)); } final IntervalFunction retryIntervalFunction = getRetryIntervalFunction(properties); @@ -189,11 +211,9 @@ private Consumer createConsumer(final String processorName) { consumerConfig.put(BOOTSTRAP_SERVERS_CONFIG, config.getProperty(KAFKA_BOOTSTRAP_SERVERS)); consumerConfig.put(CLIENT_ID_CONFIG, "%s-consumer".formatted(processorName)); consumerConfig.put(GROUP_ID_CONFIG, processorName); - consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); - consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); - consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); - final Map properties = getPassThroughProperties("%s.consumer".formatted(processorName)); + final String propertyPrefix = "%s.consumer".formatted(processorName.toLowerCase()); + final Map properties = getPassThroughProperties(propertyPrefix); for (final Map.Entry property : properties.entrySet()) { if (!ConsumerConfig.configNames().contains(property.getKey())) { LOGGER.warn("Consumer property %s was set for processor %s, but is unknown; Ignoring" @@ -204,7 +224,15 @@ private Consumer createConsumer(final String processorName) { consumerConfig.put(property.getKey(), property.getValue()); } + // Properties that MUST NOT be overwritten under any circumstance have to be applied + // AFTER pass-through properties. + consumerConfig.put(KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + consumerConfig.put(ENABLE_AUTO_COMMIT_CONFIG, false); // Commits are managed by parallel consumer + + LOGGER.debug("Creating consumer for processor %s with options %s".formatted(processorName, consumerConfig)); final var consumer = new KafkaConsumer(consumerConfig); + if (config.getPropertyAsBoolean(Config.AlpineKey.METRICS_ENABLED)) { new KafkaClientMetrics(consumer).bindTo(Metrics.getRegistry()); } @@ -230,6 +258,16 @@ private Map getPassThroughProperties(final String prefix) { return trimmedProperties; } + private static void requireValidProcessorName(final String name) { + if (name == null) { + throw new IllegalArgumentException("name must not be null"); + } + if (!PROCESSOR_NAME_PATTERN.matcher(name).matches()) { + throw new IllegalArgumentException("name is invalid; names must match the regular expression %s" + .formatted(PROCESSOR_NAME_PATTERN.pattern())); + } + } + private static IntervalFunction getRetryIntervalFunction(final Map properties) { final long initialDelayMs = Optional.ofNullable(properties.get(PROPERTY_RETRY_INITIAL_DELAY_MS)) .map(Long::parseLong) @@ -248,9 +286,9 @@ private static IntervalFunction getRetryIntervalFunction(final Map parallelConsumer, - RecordProcessingStrategy processingStrategy, - Topic topic) { + private record ManagedProcessor(ParallelStreamProcessor parallelConsumer, + RecordProcessingStrategy processingStrategy, + String topic) { } } diff --git a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java index 7b7e9112f..d1e3595c4 100644 --- a/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java +++ b/src/main/java/org/dependencytrack/health/HealthCheckInitializer.java @@ -36,8 +36,8 @@ public class HealthCheckInitializer implements ServletContextListener { @Override public void contextInitialized(final ServletContextEvent event) { LOGGER.info("Registering health checks"); - HealthCheckRegistry.getInstance().register("database", new DatabaseHealthCheck()); - HealthCheckRegistry.getInstance().register("kafka-processors", new KafkaProcessorsHealthCheck()); + HealthCheckRegistry.getInstance().register(DatabaseHealthCheck.class.getName(), new DatabaseHealthCheck()); + HealthCheckRegistry.getInstance().register(KafkaProcessorsHealthCheck.class.getName(), new KafkaProcessorsHealthCheck()); // TODO: Move this to its own initializer if it turns out to be useful LOGGER.info("Registering extra process metrics"); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 798f07b2e..f1dc9feb6 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -436,36 +436,36 @@ kafka.streams.transient.processing.exception.threshold.interval=PT30M # Optional # alpine.kafka.processor..processing.order=partition -# alpine.kafka.processor..max.batch.size= +# alpine.kafka.processor..max.batch.size=10 # alpine.kafka.processor..max.concurrency=1 # alpine.kafka.processor..retry.initial.delay.ms=1000 # alpine.kafka.processor..retry.multiplier=1 # alpine.kafka.processor..retry.randomization.factor=0.3 # alpine.kafka.processor..retry.max.delay.ms=60000 # alpine.kafka.processor..consumer.= -alpine.kafka.processor.repo-meta-result.max.concurrency=3 -alpine.kafka.processor.repo-meta-result.processing.order=partition -alpine.kafka.processor.repo-meta-result.consumer.group.id=dtrack-apiserver-processor - -alpine.kafka.processor.vuln-mirror.max.concurrency=3 -alpine.kafka.processor.vuln-mirror.processing.order=partition -alpine.kafka.processor.vuln-mirror.retry.initial.delay.ms=3000 -alpine.kafka.processor.vuln-mirror.retry.multiplier=2 -alpine.kafka.processor.vuln-mirror.retry.randomization.factor=0.3 -alpine.kafka.processor.vuln-mirror.retry.max.delay.ms=180000 -alpine.kafka.processor.vuln-mirror.consumer.group.id=dtrack-apiserver-processor - -alpine.kafka.processor.processed-vuln-scan-results.processing.order=unordered -alpine.kafka.processor.processed-vuln-scan-results.max.batch.size=500 -alpine.kafka.processor.processed-vuln-scan-results.max.concurrency=1 -alpine.kafka.processor.processed-vuln-scan-results.consumer.group.id=dtrack-apiserver-processor -alpine.kafka.processor.processed-vuln-scan-results.consumer.max.poll.records=1000 -alpine.kafka.processor.processed-vuln-scan-results.consumer.fetch.min.bytes=16384 - -alpine.kafka.processor.delayed-bom-processed-notification.processing.order=unordered -alpine.kafka.processor.delayed-bom-processed-notification.max.batch.size=100 -alpine.kafka.processor.delayed-bom-processed-notification.max.concurrency=1 -alpine.kafka.processor.delayed-bom-processed-notification.consumer.group.id=dtrack-apiserver-processor +alpine.kafka.processor.repo.meta.result.max.concurrency=3 +alpine.kafka.processor.repo.meta.result.processing.order=partition +alpine.kafka.processor.repo.meta.result.consumer.group.id=dtrack-apiserver-processor + +alpine.kafka.processor.mirrored.vuln.max.concurrency=3 +alpine.kafka.processor.mirrored.vuln.processing.order=partition +alpine.kafka.processor.mirrored.vuln.retry.initial.delay.ms=3000 +alpine.kafka.processor.mirrored.vuln.retry.multiplier=2 +alpine.kafka.processor.mirrored.vuln.retry.randomization.factor=0.3 +alpine.kafka.processor.mirrored.vuln.retry.max.delay.ms=180000 +alpine.kafka.processor.mirrored.vuln.consumer.group.id=dtrack-apiserver-processor + +alpine.kafka.processor.processed.vuln.scan.result.processing.order=unordered +alpine.kafka.processor.processed.vuln.scan.result.max.batch.size=500 +alpine.kafka.processor.processed.vuln.scan.result.max.concurrency=1 +alpine.kafka.processor.processed.vuln.scan.result.consumer.group.id=dtrack-apiserver-processor +alpine.kafka.processor.processed.vuln.scan.result.consumer.max.poll.records=1000 +alpine.kafka.processor.processed.vuln.scan.result.consumer.fetch.min.bytes=16384 + +alpine.kafka.processor.delayed.bom.processed.notification.processing.order=unordered +alpine.kafka.processor.delayed.bom.processed.notification.max.batch.size=100 +alpine.kafka.processor.delayed.bom.processed.notification.max.concurrency=1 +alpine.kafka.processor.delayed.bom.processed.notification.consumer.group.id=dtrack-apiserver-processor # Scheduling tasks after 3 minutes (3*60*1000) of starting application task.scheduler.initial.delay=180000 diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java index 47d4e1456..912eb4e83 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java @@ -67,7 +67,7 @@ public void test() throws Exception { record -> recordsProcessed.incrementAndGet(); try (final var processorManager = new RecordProcessorManager(configMock)) { - processorManager.register("foo", recordProcessor, inputTopic); + processorManager.registerProcessor("foo", recordProcessor, inputTopic); for (int i = 0; i < 100; i++) { kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo" + i, "bar" + i))) @@ -114,7 +114,7 @@ public void testSingleRecordProcessorRetry() throws Exception { )); try (final var processorManager = new RecordProcessorManager(configMock)) { - processorManager.register("foo", recordProcessor, inputTopic); + processorManager.registerProcessor("foo", recordProcessor, inputTopic); kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", "bar"))) .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) @@ -152,7 +152,7 @@ public void testBatchProcessor() throws Exception { }; try (final var processorManager = new RecordProcessorManager(configMock)) { - processorManager.register("foo", recordProcessor, inputTopic); + processorManager.registerBatchProcessor("foo", recordProcessor, inputTopic); for (int i = 0; i < 1_000; i++) { kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo" + i, "bar" + i))) From 6578f9540816879adf2f2e2ad761ff3c7101ac27 Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 30 Dec 2023 18:16:57 +0100 Subject: [PATCH 21/26] Simplify processor API class names Signed-off-by: nscuro --- ...ayedBomProcessedNotificationProcessor.java | 10 +++---- .../processor/KafkaProcessorsInitializer.java | 4 +-- ...essedVulnerabilityScanResultProcessor.java | 8 ++--- .../RepositoryMetaResultProcessor.java | 12 ++++---- .../VulnerabilityMirrorProcessor.java | 12 ++++---- ...y.java => AbstractProcessingStrategy.java} | 10 +++---- ...tegy.java => BatchProcessingStrategy.java} | 16 +++++----- ...cordProcessor.java => BatchProcessor.java} | 8 ++--- ...gStrategy.java => ProcessingStrategy.java} | 2 +- ...gleRecordProcessor.java => Processor.java} | 8 ++--- ...ssorManager.java => ProcessorManager.java} | 20 ++++++------- .../api/SingleRecordProcessingStrategy.java | 16 +++++----- ...xception.java => ProcessingException.java} | 8 ++--- .../RetryableProcessingException.java | 29 +++++++++++++++++++ .../RetryableRecordProcessingException.java | 29 ------------------- ...gerTest.java => ProcessorManagerTest.java} | 18 ++++++------ 16 files changed, 105 insertions(+), 105 deletions(-) rename src/main/java/org/dependencytrack/event/kafka/processor/api/{AbstractRecordProcessingStrategy.java => AbstractProcessingStrategy.java} (89%) rename src/main/java/org/dependencytrack/event/kafka/processor/api/{BatchRecordProcessingStrategy.java => BatchProcessingStrategy.java} (76%) rename src/main/java/org/dependencytrack/event/kafka/processor/api/{BatchRecordProcessor.java => BatchProcessor.java} (66%) rename src/main/java/org/dependencytrack/event/kafka/processor/api/{RecordProcessingStrategy.java => ProcessingStrategy.java} (90%) rename src/main/java/org/dependencytrack/event/kafka/processor/api/{SingleRecordProcessor.java => Processor.java} (57%) rename src/main/java/org/dependencytrack/event/kafka/processor/api/{RecordProcessorManager.java => ProcessorManager.java} (95%) rename src/main/java/org/dependencytrack/event/kafka/processor/exception/{RecordProcessingException.java => ProcessingException.java} (56%) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableProcessingException.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java rename src/test/java/org/dependencytrack/event/kafka/processor/api/{RecordProcessorManagerTest.java => ProcessorManagerTest.java} (90%) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java index 13828a9d2..ae86c7be8 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java @@ -11,8 +11,8 @@ import org.dependencytrack.event.kafka.KafkaEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.event.kafka.processor.api.BatchRecordProcessor; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.api.BatchProcessor; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import org.dependencytrack.model.VulnerabilityScan; import org.dependencytrack.notification.NotificationConstants; import org.dependencytrack.notification.NotificationGroup; @@ -33,13 +33,13 @@ import static org.dependencytrack.proto.notification.v1.Scope.SCOPE_PORTFOLIO; /** - * A {@link BatchRecordProcessor} responsible for dispatching {@link NotificationGroup#BOM_PROCESSED} notifications + * A {@link BatchProcessor} responsible for dispatching {@link NotificationGroup#BOM_PROCESSED} notifications * upon detection of a completed {@link VulnerabilityScan}. *

* The completion detection is based on {@link NotificationGroup#PROJECT_VULN_ANALYSIS_COMPLETE} notifications. * This processor does nothing unless {@link ConfigKey#TMP_DELAY_BOM_PROCESSED_NOTIFICATION} is enabled. */ -public class DelayedBomProcessedNotificationProcessor implements BatchRecordProcessor { +public class DelayedBomProcessedNotificationProcessor implements BatchProcessor { public static final String PROCESSOR_NAME = "delayed.bom.processed.notification"; @@ -58,7 +58,7 @@ public DelayedBomProcessedNotificationProcessor() { } @Override - public void process(final List> records) throws RecordProcessingException { + public void process(final List> records) throws ProcessingException { if (!config.getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION)) { return; } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java index fd558e4aa..0242a3c45 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java @@ -4,7 +4,7 @@ import alpine.common.logging.Logger; import org.dependencytrack.common.ConfigKey; import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.event.kafka.processor.api.RecordProcessorManager; +import org.dependencytrack.event.kafka.processor.api.ProcessorManager; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -13,7 +13,7 @@ public class KafkaProcessorsInitializer implements ServletContextListener { private static final Logger LOGGER = Logger.getLogger(KafkaProcessorsInitializer.class); - static final RecordProcessorManager PROCESSOR_MANAGER = new RecordProcessorManager(); + static final ProcessorManager PROCESSOR_MANAGER = new ProcessorManager(); @Override public void contextInitialized(final ServletContextEvent event) { diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java index 533b44c61..7fb243a4c 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java @@ -7,8 +7,8 @@ import org.dependencytrack.event.ComponentPolicyEvaluationEvent; import org.dependencytrack.event.ProjectMetricsUpdateEvent; import org.dependencytrack.event.ProjectPolicyEvaluationEvent; -import org.dependencytrack.event.kafka.processor.api.BatchRecordProcessor; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.api.BatchProcessor; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import org.dependencytrack.model.VulnerabilityScan; import org.dependencytrack.model.VulnerabilityScan.Status; import org.dependencytrack.persistence.QueryManager; @@ -27,12 +27,12 @@ import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi; import static org.dependencytrack.proto.vulnanalysis.v1.ScanStatus.SCAN_STATUS_FAILED; -public class ProcessedVulnerabilityScanResultProcessor implements BatchRecordProcessor { +public class ProcessedVulnerabilityScanResultProcessor implements BatchProcessor { public static final String PROCESSOR_NAME = "processed.vuln.scan.result"; @Override - public void process(final List> records) throws RecordProcessingException { + public void process(final List> records) throws ProcessingException { final List completedScans = processScanResults(records); triggerPolicyEvalAndMetricsUpdates(completedScans); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index 6c67b723b..2697e3ad9 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -7,8 +7,8 @@ import io.micrometer.core.instrument.Timer; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.api.Processor; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import org.dependencytrack.model.FetchStatus; import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.RepositoryMetaComponent; @@ -28,9 +28,9 @@ import static org.dependencytrack.event.kafka.componentmeta.IntegrityCheck.performIntegrityCheck; /** - * A {@link SingleRecordProcessor} that ingests repository metadata {@link AnalysisResult}s. + * A {@link Processor} that ingests repository metadata {@link AnalysisResult}s. */ -public class RepositoryMetaResultProcessor implements SingleRecordProcessor { +public class RepositoryMetaResultProcessor implements Processor { public static final String PROCESSOR_NAME = "repo.meta.result"; @@ -40,7 +40,7 @@ public class RepositoryMetaResultProcessor implements SingleRecordProcessor record) throws RecordProcessingException { + public void process(final ConsumerRecord record) throws ProcessingException { final Timer.Sample timerSample = Timer.start(); if (!isRecordValid(record)) { return; @@ -52,7 +52,7 @@ public void process(final ConsumerRecord record) throws performIntegrityCheck(integrityMetaComponent, record.value(), qm); } } catch (Exception e) { - throw new RecordProcessingException("An unexpected error occurred while processing record %s".formatted(record), e); + throw new ProcessingException("An unexpected error occurred while processing record %s".formatted(record), e); } finally { timerSample.stop(TIMER); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java index 2c5bb078d..632a8ffa8 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java @@ -10,8 +10,8 @@ import org.cyclonedx.proto.v1_4.Bom; import org.cyclonedx.proto.v1_4.Component; import org.cyclonedx.proto.v1_4.VulnerabilityAffects; -import org.dependencytrack.event.kafka.processor.api.SingleRecordProcessor; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.api.Processor; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import org.dependencytrack.model.Vulnerability; import org.dependencytrack.model.VulnerableSoftware; import org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln; @@ -29,9 +29,9 @@ import java.util.Optional; /** - * A {@link SingleRecordProcessor} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. + * A {@link Processor} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. */ -public class VulnerabilityMirrorProcessor implements SingleRecordProcessor { +public class VulnerabilityMirrorProcessor implements Processor { public static final String PROCESSOR_NAME = "mirrored.vuln"; @@ -41,7 +41,7 @@ public class VulnerabilityMirrorProcessor implements SingleRecordProcessor record) throws RecordProcessingException { + public void process(final ConsumerRecord record) throws ProcessingException { final Timer.Sample timerSample = Timer.start(); try (QueryManager qm = new QueryManager().withL2CacheDisabled()) { @@ -119,7 +119,7 @@ public void process(final ConsumerRecord record) throws RecordProce qm.persist(synchronizedVulnerability); // Event.dispatch(new IndexEvent(IndexEvent.Action.COMMIT, Vulnerability.class)); } catch (Exception e) { - throw new RecordProcessingException("Synchronizing vulnerability %s failed".formatted(record.key()), e); + throw new ProcessingException("Synchronizing vulnerability %s failed".formatted(record.key()), e); } finally { timerSample.stop(TIMER); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractProcessingStrategy.java similarity index 89% rename from src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java rename to src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractProcessingStrategy.java index 3de47153e..27d198b56 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/AbstractProcessingStrategy.java @@ -7,7 +7,7 @@ import org.apache.kafka.common.serialization.Serde; import org.datanucleus.api.jdo.exceptions.ConnectionInUseException; import org.datanucleus.store.query.QueryInterruptedException; -import org.dependencytrack.event.kafka.processor.exception.RetryableRecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.RetryableProcessingException; import org.postgresql.util.PSQLState; import javax.jdo.JDOOptimisticVerificationException; @@ -19,17 +19,17 @@ import java.util.concurrent.TimeoutException; /** - * An abstract {@link RecordProcessingStrategy} that provides various shared functionality. + * An abstract {@link ProcessingStrategy} that provides various shared functionality. * * @param Type of the {@link ConsumerRecord} key * @param Type of the {@link ConsumerRecord} value */ -abstract class AbstractRecordProcessingStrategy implements RecordProcessingStrategy { +abstract class AbstractProcessingStrategy implements ProcessingStrategy { private final Serde keySerde; private final Serde valueSerde; - AbstractRecordProcessingStrategy(final Serde keySerde, final Serde valueSerde) { + AbstractProcessingStrategy(final Serde keySerde, final Serde valueSerde) { this.keySerde = keySerde; this.valueSerde = valueSerde; } @@ -70,7 +70,7 @@ ConsumerRecord deserialize(final ConsumerRecord record) { ); boolean isRetryableException(final Throwable throwable) { - if (throwable instanceof RetryableRecordProcessingException) { + if (throwable instanceof RetryableProcessingException) { return true; } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessingStrategy.java similarity index 76% rename from src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java rename to src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessingStrategy.java index bad98d165..343e8aa7e 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessingStrategy.java @@ -5,25 +5,25 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import java.util.ArrayList; import java.util.List; /** - * A {@link RecordProcessingStrategy} that processes records in batches. + * A {@link ProcessingStrategy} that processes records in batches. * * @param Type of the {@link ConsumerRecord} key * @param Type of the {@link ConsumerRecord} value */ -class BatchRecordProcessingStrategy extends AbstractRecordProcessingStrategy { +class BatchProcessingStrategy extends AbstractProcessingStrategy { - private static final Logger LOGGER = Logger.getLogger(BatchRecordProcessingStrategy.class); + private static final Logger LOGGER = Logger.getLogger(BatchProcessingStrategy.class); - private final BatchRecordProcessor batchConsumer; + private final BatchProcessor batchConsumer; - BatchRecordProcessingStrategy(final BatchRecordProcessor batchConsumer, - final Serde keySerde, final Serde valueSerde) { + BatchProcessingStrategy(final BatchProcessor batchConsumer, + final Serde keySerde, final Serde valueSerde) { super(keySerde, valueSerde); this.batchConsumer = batchConsumer; } @@ -50,7 +50,7 @@ public void processRecords(final List> records) { try { batchConsumer.process(deserializedRecords); - } catch (RecordProcessingException | RuntimeException e) { + } catch (ProcessingException | RuntimeException e) { if (isRetryableException(e)) { LOGGER.warn("Encountered retryable exception while processing %d records".formatted(deserializedRecords.size()), e); throw new PCRetriableException(e); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java similarity index 66% rename from src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java index d9707d593..ee1a00e81 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchRecordProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java @@ -1,7 +1,7 @@ package org.dependencytrack.event.kafka.processor.api; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import java.util.List; @@ -11,14 +11,14 @@ * @param Type of the {@link ConsumerRecord} key * @param Type of the {@link ConsumerRecord} value */ -public interface BatchRecordProcessor { +public interface BatchProcessor { /** * Process a batch of {@link ConsumerRecord}s. * * @param records Batch of {@link ConsumerRecord}s to process - * @throws RecordProcessingException When consuming the batch of {@link ConsumerRecord}s failed + * @throws ProcessingException When consuming the batch of {@link ConsumerRecord}s failed */ - void process(final List> records) throws RecordProcessingException; + void process(final List> records) throws ProcessingException; } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessingStrategy.java similarity index 90% rename from src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java rename to src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessingStrategy.java index 475928bf5..459f80078 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessingStrategy.java @@ -4,7 +4,7 @@ import java.util.List; -interface RecordProcessingStrategy { +interface ProcessingStrategy { /** * Process zero or more {@link ConsumerRecord}s. diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java similarity index 57% rename from src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java index 3ecf726b0..6bf98284a 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java @@ -1,7 +1,7 @@ package org.dependencytrack.event.kafka.processor.api; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; /** * A processor of individual {@link ConsumerRecord}s. @@ -9,14 +9,14 @@ * @param Type of the {@link ConsumerRecord} key * @param Type of the {@link ConsumerRecord} value */ -public interface SingleRecordProcessor { +public interface Processor { /** * Process a {@link ConsumerRecord}. * * @param record The {@link ConsumerRecord} to process - * @throws RecordProcessingException When processing the {@link ConsumerRecord} failed + * @throws ProcessingException When processing the {@link ConsumerRecord} failed */ - void process(final ConsumerRecord record) throws RecordProcessingException; + void process(final ConsumerRecord record) throws ProcessingException; } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java similarity index 95% rename from src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java rename to src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java index b170736b2..4916c87a6 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManager.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorManager.java @@ -34,9 +34,9 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.dependencytrack.common.ConfigKey.KAFKA_BOOTSTRAP_SERVERS; -public class RecordProcessorManager implements AutoCloseable { +public class ProcessorManager implements AutoCloseable { - private static final Logger LOGGER = Logger.getLogger(RecordProcessorManager.class); + private static final Logger LOGGER = Logger.getLogger(ProcessorManager.class); private static final Pattern PROCESSOR_NAME_PATTERN = Pattern.compile("^[a-z.]+$"); private static final String PROPERTY_MAX_BATCH_SIZE = "max.batch.size"; @@ -57,16 +57,16 @@ public class RecordProcessorManager implements AutoCloseable { private final Map managedProcessors = new LinkedHashMap<>(); private final Config config; - public RecordProcessorManager() { + public ProcessorManager() { this(Config.getInstance()); } - public RecordProcessorManager(final Config config) { + public ProcessorManager(final Config config) { this.config = config; } /** - * Register a new {@link SingleRecordProcessor}. + * Register a new {@link Processor}. * * @param name Name of the processor to register * @param processor The processor to register @@ -74,7 +74,7 @@ public RecordProcessorManager(final Config config) { * @param Type of record keys in the topic * @param Type of record values in the topic */ - public void registerProcessor(final String name, final SingleRecordProcessor processor, final Topic topic) { + public void registerProcessor(final String name, final Processor processor, final Topic topic) { requireValidProcessorName(name); final var processingStrategy = new SingleRecordProcessingStrategy<>(processor, topic.keySerde(), topic.valueSerde()); final ParallelStreamProcessor parallelConsumer = createParallelConsumer(name, false); @@ -82,7 +82,7 @@ public void registerProcessor(final String name, final SingleRecordProces } /** - * Register a new {@link BatchRecordProcessor}. + * Register a new {@link BatchProcessor}. * * @param name Name of the processor to register * @param processor The processor to register @@ -90,9 +90,9 @@ public void registerProcessor(final String name, final SingleRecordProces * @param Type of record keys in the topic * @param Type of record values in the topic */ - public void registerBatchProcessor(final String name, final BatchRecordProcessor processor, final Topic topic) { + public void registerBatchProcessor(final String name, final BatchProcessor processor, final Topic topic) { requireValidProcessorName(name); - final var processingStrategy = new BatchRecordProcessingStrategy<>(processor, topic.keySerde(), topic.valueSerde()); + final var processingStrategy = new BatchProcessingStrategy<>(processor, topic.keySerde(), topic.valueSerde()); final ParallelStreamProcessor parallelConsumer = createParallelConsumer(name, true); managedProcessors.put(name, new ManagedProcessor(parallelConsumer, processingStrategy, topic.name())); } @@ -287,7 +287,7 @@ private static IntervalFunction getRetryIntervalFunction(final Map parallelConsumer, - RecordProcessingStrategy processingStrategy, + ProcessingStrategy processingStrategy, String topic) { } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java index 352e90c24..247e65943 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/SingleRecordProcessingStrategy.java @@ -5,26 +5,26 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Serde; -import org.dependencytrack.event.kafka.processor.exception.RecordProcessingException; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import java.util.List; /** - * A {@link RecordProcessingStrategy} that processes records individually. + * A {@link ProcessingStrategy} that processes records individually. * * @param Type of the {@link ConsumerRecord} key * @param Type of the {@link ConsumerRecord} value */ -class SingleRecordProcessingStrategy extends AbstractRecordProcessingStrategy { +class SingleRecordProcessingStrategy extends AbstractProcessingStrategy { private static final Logger LOGGER = Logger.getLogger(SingleRecordProcessingStrategy.class); - private final SingleRecordProcessor recordProcessor; + private final Processor processor; - SingleRecordProcessingStrategy(final SingleRecordProcessor recordProcessor, + SingleRecordProcessingStrategy(final Processor processor, final Serde keySerde, final Serde valueSerde) { super(keySerde, valueSerde); - this.recordProcessor = recordProcessor; + this.processor = processor; } /** @@ -51,8 +51,8 @@ public void processRecords(final List> records) { } try { - recordProcessor.process(deserializedRecord); - } catch (RecordProcessingException | RuntimeException e) { + processor.process(deserializedRecord); + } catch (ProcessingException | RuntimeException e) { if (isRetryableException(e)) { LOGGER.warn("Encountered retryable exception while processing %s".formatted(deserializedRecord), e); throw new PCRetriableException(e); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java b/src/main/java/org/dependencytrack/event/kafka/processor/exception/ProcessingException.java similarity index 56% rename from src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java rename to src/main/java/org/dependencytrack/event/kafka/processor/exception/ProcessingException.java index ba13541df..56dbb56e0 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RecordProcessingException.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/exception/ProcessingException.java @@ -3,26 +3,26 @@ /** * An {@link Exception} indicating an error during record processing. */ -public class RecordProcessingException extends Exception { +public class ProcessingException extends Exception { /** * {@inheritDoc} */ - public RecordProcessingException(final String message) { + public ProcessingException(final String message) { super(message); } /** * {@inheritDoc} */ - public RecordProcessingException(final String message, final Throwable cause) { + public ProcessingException(final String message, final Throwable cause) { super(message, cause); } /** * {@inheritDoc} */ - public RecordProcessingException(final Throwable cause) { + public ProcessingException(final Throwable cause) { super(cause); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableProcessingException.java b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableProcessingException.java new file mode 100644 index 000000000..1dd51fb75 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableProcessingException.java @@ -0,0 +1,29 @@ +package org.dependencytrack.event.kafka.processor.exception; + +/** + * A {@link ProcessingException} indicating a retryable error. + */ +public class RetryableProcessingException extends ProcessingException { + + /** + * {@inheritDoc} + */ + public RetryableProcessingException(final String message) { + super(message); + } + + /** + * {@inheritDoc} + */ + public RetryableProcessingException(final String message, final Throwable cause) { + super(message, cause); + } + + /** + * {@inheritDoc} + */ + public RetryableProcessingException(final Throwable cause) { + super(cause); + } + +} diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java b/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java deleted file mode 100644 index 87af27a93..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/processor/exception/RetryableRecordProcessingException.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.dependencytrack.event.kafka.processor.exception; - -/** - * A {@link RecordProcessingException} indicating a retryable error. - */ -public class RetryableRecordProcessingException extends RecordProcessingException { - - /** - * {@inheritDoc} - */ - public RetryableRecordProcessingException(final String message) { - super(message); - } - - /** - * {@inheritDoc} - */ - public RetryableRecordProcessingException(final String message, final Throwable cause) { - super(message, cause); - } - - /** - * {@inheritDoc} - */ - public RetryableRecordProcessingException(final Throwable cause) { - super(cause); - } - -} diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/api/ProcessorManagerTest.java similarity index 90% rename from src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java rename to src/test/java/org/dependencytrack/event/kafka/processor/api/ProcessorManagerTest.java index 912eb4e83..0602fd4f6 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/api/RecordProcessorManagerTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/api/ProcessorManagerTest.java @@ -31,7 +31,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -public class RecordProcessorManagerTest { +public class ProcessorManagerTest { @Rule public RedpandaContainer kafkaContainer = new RedpandaContainer(DockerImageName @@ -63,11 +63,11 @@ public void test() throws Exception { "kafka.processor.foo.consumer.auto.offset.reset", "earliest" )); - final SingleRecordProcessor recordProcessor = + final Processor processor = record -> recordsProcessed.incrementAndGet(); - try (final var processorManager = new RecordProcessorManager(configMock)) { - processorManager.registerProcessor("foo", recordProcessor, inputTopic); + try (final var processorManager = new ProcessorManager(configMock)) { + processorManager.registerProcessor("foo", processor, inputTopic); for (int i = 0; i < 100; i++) { kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo" + i, "bar" + i))) @@ -97,7 +97,7 @@ public void testSingleRecordProcessorRetry() throws Exception { .thenThrow(new RuntimeException(new TimeoutException())) .thenReturn("done"); - final SingleRecordProcessor recordProcessor = record -> { + final Processor processor = record -> { attemptsCounter.incrementAndGet(); objectSpy.toString(); }; @@ -113,8 +113,8 @@ public void testSingleRecordProcessorRetry() throws Exception { "kafka.processor.foo.consumer.auto.offset.reset", "earliest" )); - try (final var processorManager = new RecordProcessorManager(configMock)) { - processorManager.registerProcessor("foo", recordProcessor, inputTopic); + try (final var processorManager = new ProcessorManager(configMock)) { + processorManager.registerProcessor("foo", processor, inputTopic); kafka.send(SendKeyValues.to("input", List.of(new KeyValue<>("foo", "bar"))) .with(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) @@ -146,12 +146,12 @@ public void testBatchProcessor() throws Exception { "kafka.processor.foo.consumer.auto.offset.reset", "earliest" )); - final BatchRecordProcessor recordProcessor = records -> { + final BatchProcessor recordProcessor = records -> { recordsProcessed.addAndGet(records.size()); actualBatchSizes.add(records.size()); }; - try (final var processorManager = new RecordProcessorManager(configMock)) { + try (final var processorManager = new ProcessorManager(configMock)) { processorManager.registerBatchProcessor("foo", recordProcessor, inputTopic); for (int i = 0; i < 1_000; i++) { From e071090b4058a202ebfc4496ed6cf263af01f05f Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 30 Dec 2023 18:36:35 +0100 Subject: [PATCH 22/26] Minor tweaks Signed-off-by: nscuro --- ...DelayedBomProcessedNotificationProcessor.java | 7 ++++++- .../processor/KafkaProcessorsInitializer.java | 8 ++++---- ....java => MirroredVulnerabilityProcessor.java} | 6 +++--- ...rocessedVulnerabilityScanResultProcessor.java | 16 +++++++++++++++- .../processor/RepositoryMetaResultProcessor.java | 2 +- ...a => MirroredVulnerabilityProcessorTest.java} | 16 ++++++++-------- 6 files changed, 37 insertions(+), 18 deletions(-) rename src/main/java/org/dependencytrack/event/kafka/processor/{VulnerabilityMirrorProcessor.java => MirroredVulnerabilityProcessor.java} (98%) rename src/test/java/org/dependencytrack/event/kafka/processor/{VulnerabilityMirrorProcessorTest.java => MirroredVulnerabilityProcessorTest.java} (98%) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java index ae86c7be8..f87742df7 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java @@ -41,7 +41,7 @@ */ public class DelayedBomProcessedNotificationProcessor implements BatchProcessor { - public static final String PROCESSOR_NAME = "delayed.bom.processed.notification"; + static final String PROCESSOR_NAME = "delayed.bom.processed.notification"; private static final Logger LOGGER = Logger.getLogger(DelayedBomProcessedNotificationProcessor.class); @@ -119,6 +119,11 @@ private void dispatchNotifications(final List sub } eventDispatcher.dispatchAllBlocking(events); + + for (final BomConsumedOrProcessedSubject subject : subjects) { + LOGGER.info("Dispatched delayed %s notification (token=%s, project=%s)" + .formatted(GROUP_BOM_PROCESSED, subject.getToken(), subject.getProject().getUuid())); + } } } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java index 0242a3c45..f1218e87e 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java @@ -17,10 +17,10 @@ public class KafkaProcessorsInitializer implements ServletContextListener { @Override public void contextInitialized(final ServletContextEvent event) { - LOGGER.info("Initializing Kafka processors"); + LOGGER.info("Initializing processors"); - PROCESSOR_MANAGER.registerProcessor(VulnerabilityMirrorProcessor.PROCESSOR_NAME, - new VulnerabilityMirrorProcessor(), KafkaTopics.NEW_VULNERABILITY); + PROCESSOR_MANAGER.registerProcessor(MirroredVulnerabilityProcessor.PROCESSOR_NAME, + new MirroredVulnerabilityProcessor(), KafkaTopics.NEW_VULNERABILITY); PROCESSOR_MANAGER.registerProcessor(RepositoryMetaResultProcessor.PROCESSOR_NAME, new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); PROCESSOR_MANAGER.registerBatchProcessor(ProcessedVulnerabilityScanResultProcessor.PROCESSOR_NAME, @@ -35,7 +35,7 @@ public void contextInitialized(final ServletContextEvent event) { @Override public void contextDestroyed(final ServletContextEvent event) { - LOGGER.info("Stopping Kafka processors"); + LOGGER.info("Stopping processors"); PROCESSOR_MANAGER.close(); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java similarity index 98% rename from src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java rename to src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java index 632a8ffa8..c1418d075 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java @@ -31,11 +31,11 @@ /** * A {@link Processor} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. */ -public class VulnerabilityMirrorProcessor implements Processor { +public class MirroredVulnerabilityProcessor implements Processor { - public static final String PROCESSOR_NAME = "mirrored.vuln"; + static final String PROCESSOR_NAME = "mirrored.vuln"; - private static final Logger LOGGER = Logger.getLogger(VulnerabilityMirrorProcessor.class); + private static final Logger LOGGER = Logger.getLogger(MirroredVulnerabilityProcessor.class); private static final Timer TIMER = Timer.builder("vuln_mirror_processing") .description("Time taken to process mirrored vulnerabilities") .register(Metrics.getRegistry()); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java index 7fb243a4c..766067c63 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java @@ -1,5 +1,6 @@ package org.dependencytrack.event.kafka.processor; +import alpine.common.logging.Logger; import alpine.event.framework.ChainableEvent; import alpine.event.framework.Event; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -27,13 +28,24 @@ import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi; import static org.dependencytrack.proto.vulnanalysis.v1.ScanStatus.SCAN_STATUS_FAILED; +/** + * A {@link BatchProcessor} that records successfully processed {@link ScanResult}s for their respective + * {@link VulnerabilityScan}, and triggers follow-up processes in case a scan is complete. + */ public class ProcessedVulnerabilityScanResultProcessor implements BatchProcessor { - public static final String PROCESSOR_NAME = "processed.vuln.scan.result"; + static final String PROCESSOR_NAME = "processed.vuln.scan.result"; + + private static final Logger LOGGER = Logger.getLogger(ProcessedVulnerabilityScanResultProcessor.class); @Override public void process(final List> records) throws ProcessingException { final List completedScans = processScanResults(records); + if (!completedScans.isEmpty() && LOGGER.isDebugEnabled()) { + LOGGER.debug("Detected completion of %s vulnerability scans: %s" + .formatted(completedScans.size(), completedScans.stream().map(VulnerabilityScan::getToken).toList())); + } + triggerPolicyEvalAndMetricsUpdates(completedScans); } @@ -128,6 +140,8 @@ private static List recordScanResults(final Handle jdbiHandle // does not allow a WHERE clause. Tried using a CTE as workaround: // WITH "CTE" AS (UPDATE ... RETURNING ...) SELECT * FROM "CTE" // but that didn't return any results at all. + // The good news is that the query typically modifies only a handful + // of scans, so we're wasting not too many resources here. .filter(vulnScan -> vulnScan.getStatus() == Status.COMPLETED || vulnScan.getStatus() == Status.FAILED) .toList(); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index 2697e3ad9..e0267c858 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -32,7 +32,7 @@ */ public class RepositoryMetaResultProcessor implements Processor { - public static final String PROCESSOR_NAME = "repo.meta.result"; + static final String PROCESSOR_NAME = "repo.meta.result"; private static final Logger LOGGER = Logger.getLogger(RepositoryMetaResultProcessor.class); private static final Timer TIMER = Timer.builder("repo_meta_result_processing") diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java similarity index 98% rename from src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java index 701eddd1b..4dcde9801 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityMirrorProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java @@ -9,7 +9,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.dependencytrack.util.KafkaTestUtil.generateBomFromJson; -public class VulnerabilityMirrorProcessorTest extends AbstractProcessorTest { +public class MirroredVulnerabilityProcessorTest extends AbstractProcessorTest { @Before public void setUp() throws Exception { @@ -64,7 +64,7 @@ public void testProcessNvdVuln() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); @@ -199,7 +199,7 @@ public void testProcessGitHubVuln() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("GITHUB/GHSA-fxwm-579q-49qq", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-fxwm-579q-49qq"); @@ -406,7 +406,7 @@ public void testProcessOsvVuln() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("OSV/GHSA-2cc5-23r7-vc4v", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-2cc5-23r7-vc4v"); @@ -555,7 +555,7 @@ public void testProcessVulnWithoutAffects() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); @@ -624,7 +624,7 @@ public void testProcessVulnWithUnmatchedAffectsBomRef() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); @@ -719,7 +719,7 @@ public void testProcessVulnWithVersConstraints() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); @@ -988,7 +988,7 @@ public void testProcessVulnWithInvalidCpeOrPurl() throws Exception { } """; - final var processor = new VulnerabilityMirrorProcessor(); + final var processor = new MirroredVulnerabilityProcessor(); processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); From ce20a62df534f50ea94c36aaf48fc30f116006de Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 30 Dec 2023 18:45:47 +0100 Subject: [PATCH 23/26] Make processor implementations package-private Signed-off-by: nscuro --- .../processor/DelayedBomProcessedNotificationProcessor.java | 4 ++-- .../event/kafka/processor/MirroredVulnerabilityProcessor.java | 2 +- .../processor/ProcessedVulnerabilityScanResultProcessor.java | 2 +- .../event/kafka/processor/RepositoryMetaResultProcessor.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java index f87742df7..4e1b33779 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java @@ -39,7 +39,7 @@ * The completion detection is based on {@link NotificationGroup#PROJECT_VULN_ANALYSIS_COMPLETE} notifications. * This processor does nothing unless {@link ConfigKey#TMP_DELAY_BOM_PROCESSED_NOTIFICATION} is enabled. */ -public class DelayedBomProcessedNotificationProcessor implements BatchProcessor { +class DelayedBomProcessedNotificationProcessor implements BatchProcessor { static final String PROCESSOR_NAME = "delayed.bom.processed.notification"; @@ -48,7 +48,7 @@ public class DelayedBomProcessedNotificationProcessor implements BatchProcessor< private final Config config; private final KafkaEventDispatcher eventDispatcher; - public DelayedBomProcessedNotificationProcessor() { + DelayedBomProcessedNotificationProcessor() { this(Config.getInstance(), new KafkaEventDispatcher()); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java index c1418d075..5b5f21670 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessor.java @@ -31,7 +31,7 @@ /** * A {@link Processor} that ingests vulnerability data from CycloneDX Bill of Vulnerabilities. */ -public class MirroredVulnerabilityProcessor implements Processor { +class MirroredVulnerabilityProcessor implements Processor { static final String PROCESSOR_NAME = "mirrored.vuln"; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java index 766067c63..47c9a2175 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessedVulnerabilityScanResultProcessor.java @@ -32,7 +32,7 @@ * A {@link BatchProcessor} that records successfully processed {@link ScanResult}s for their respective * {@link VulnerabilityScan}, and triggers follow-up processes in case a scan is complete. */ -public class ProcessedVulnerabilityScanResultProcessor implements BatchProcessor { +class ProcessedVulnerabilityScanResultProcessor implements BatchProcessor { static final String PROCESSOR_NAME = "processed.vuln.scan.result"; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index e0267c858..077862093 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -30,7 +30,7 @@ /** * A {@link Processor} that ingests repository metadata {@link AnalysisResult}s. */ -public class RepositoryMetaResultProcessor implements Processor { +class RepositoryMetaResultProcessor implements Processor { static final String PROCESSOR_NAME = "repo.meta.result"; From 80b5742027899f282c5ffb1965a8d3beabfe537c Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 1 Jan 2024 16:34:52 +0100 Subject: [PATCH 24/26] Support dispatch of multiple events with arbitrary key-value types Signed-off-by: nscuro --- .../dependencytrack/event/kafka/KafkaEventDispatcher.java | 8 ++++---- .../DelayedBomProcessedNotificationProcessor.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java index 564edaddf..5865ce856 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEventDispatcher.java @@ -151,11 +151,11 @@ private Future dispatchAsyncInternal(final KafkaEvent new KafkaDefaultProducerCallback(LOGGER, event.topic().name(), event.key()))); } - public void dispatchAllBlocking(final List> events) { + public void dispatchAllBlocking(final List> events) { dispatchAllBlocking(events, null); } - public void dispatchAllBlocking(final List> events, Callback callback) { + public void dispatchAllBlocking(final List> events, Callback callback) { final var countDownLatch = new CountDownLatch(events.size()); callback = requireNonNullElseGet(callback, () -> new KafkaDefaultProducerCallback(LOGGER)); @@ -174,9 +174,9 @@ public void dispatchAllBlocking(final List> events, Call } } - public List> dispatchAllAsync(final List> events, Callback callback) { + public List> dispatchAllAsync(final List> events, Callback callback) { final var records = new ArrayList>(events.size()); - for (final KafkaEvent event : events) { + for (final KafkaEvent event : events) { records.add(toProducerRecord(event)); } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java index 4e1b33779..ba1c9ab1a 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/DelayedBomProcessedNotificationProcessor.java @@ -103,7 +103,7 @@ private static Set extractTokens(final List subjects) { final Timestamp timestamp = Timestamps.now(); - final var events = new ArrayList>(subjects.size()); + final var events = new ArrayList>(subjects.size()); for (final BomConsumedOrProcessedSubject subject : subjects) { final var event = new KafkaEvent<>(KafkaTopics.NOTIFICATION_BOM, subject.getProject().getUuid(), Notification.newBuilder() From 8f8d76a9bbd7953ef2f6a30d70004962c80b88f2 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 1 Jan 2024 16:35:12 +0100 Subject: [PATCH 25/26] Add thread safety notice Signed-off-by: nscuro --- .../event/kafka/processor/api/BatchProcessor.java | 2 ++ .../dependencytrack/event/kafka/processor/api/Processor.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java index ee1a00e81..5e1b6ff03 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/BatchProcessor.java @@ -15,6 +15,8 @@ public interface BatchProcessor { /** * Process a batch of {@link ConsumerRecord}s. + *

+ * This method may be called by multiple threads concurrently and thus MUST be thread safe! * * @param records Batch of {@link ConsumerRecord}s to process * @throws ProcessingException When consuming the batch of {@link ConsumerRecord}s failed diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java index 6bf98284a..e905a7937 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/Processor.java @@ -13,6 +13,8 @@ public interface Processor { /** * Process a {@link ConsumerRecord}. + *

+ * This method may be called by multiple threads concurrently and thus MUST be thread safe! * * @param record The {@link ConsumerRecord} to process * @throws ProcessingException When processing the {@link ConsumerRecord} failed From 530bccd195624233696ac07e5fbf0c3d40e56fa4 Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 1 Jan 2024 17:07:11 +0100 Subject: [PATCH 26/26] Migrate `VulnerabilityScanResultProcessor` Signed-off-by: nscuro --- .../event/kafka/KafkaEventConverter.java | 4 +- .../processor/KafkaProcessorsInitializer.java | 2 + .../VulnerabilityScanResultProcessor.java | 1150 +++++++++++++++++ .../processor/AbstractProcessorTest.java | 47 +- .../MirroredVulnerabilityProcessorTest.java | 14 +- .../RepositoryMetaResultProcessorTest.java | 34 +- .../VulnerabilityScanResultProcessorTest.java | 174 +-- 7 files changed, 1311 insertions(+), 114 deletions(-) create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java rename src/test/java/org/dependencytrack/event/kafka/{streams => }/processor/VulnerabilityScanResultProcessorTest.java (89%) diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java index f38b5f0ef..2cc1a0d46 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java @@ -17,7 +17,7 @@ * Utility class to convert {@link alpine.event.framework.Event}s and {@link alpine.notification.Notification}s * to {@link KafkaEvent}s. */ -final class KafkaEventConverter { +public final class KafkaEventConverter { private KafkaEventConverter() { } @@ -65,7 +65,7 @@ static KafkaEvent convert(final ComponentRepositoryMeta return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null); } - static KafkaEvent convert(final String key, final Notification notification) { + public static KafkaEvent convert(final String key, final Notification notification) { final Topic topic = switch (notification.getGroup()) { case GROUP_CONFIGURATION -> KafkaTopics.NOTIFICATION_CONFIGURATION; case GROUP_DATASOURCE_MIRRORING -> KafkaTopics.NOTIFICATION_DATASOURCE_MIRRORING; diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java index f1218e87e..ce6749dfb 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/KafkaProcessorsInitializer.java @@ -23,6 +23,8 @@ public void contextInitialized(final ServletContextEvent event) { new MirroredVulnerabilityProcessor(), KafkaTopics.NEW_VULNERABILITY); PROCESSOR_MANAGER.registerProcessor(RepositoryMetaResultProcessor.PROCESSOR_NAME, new RepositoryMetaResultProcessor(), KafkaTopics.REPO_META_ANALYSIS_RESULT); + PROCESSOR_MANAGER.registerProcessor(VulnerabilityScanResultProcessor.PROCESSOR_NAME, + new VulnerabilityScanResultProcessor(), KafkaTopics.VULN_ANALYSIS_RESULT); PROCESSOR_MANAGER.registerBatchProcessor(ProcessedVulnerabilityScanResultProcessor.PROCESSOR_NAME, new ProcessedVulnerabilityScanResultProcessor(), KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED); if (Config.getInstance().getPropertyAsBoolean(ConfigKey.TMP_DELAY_BOM_PROCESSED_NOTIFICATION)) { diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java new file mode 100644 index 000000000..3385701fc --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessor.java @@ -0,0 +1,1150 @@ +package org.dependencytrack.event.kafka.processor; + +import alpine.Config; +import alpine.common.logging.Logger; +import com.google.protobuf.Any; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.event.PortfolioVulnerabilityAnalysisEvent; +import org.dependencytrack.event.kafka.KafkaEvent; +import org.dependencytrack.event.kafka.KafkaEventConverter; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; +import org.dependencytrack.event.kafka.KafkaEventHeaders; +import org.dependencytrack.event.kafka.KafkaTopics; +import org.dependencytrack.event.kafka.KafkaUtil; +import org.dependencytrack.event.kafka.processor.api.Processor; +import org.dependencytrack.event.kafka.processor.exception.ProcessingException; +import org.dependencytrack.model.AnalysisJustification; +import org.dependencytrack.model.AnalysisResponse; +import org.dependencytrack.model.AnalysisState; +import org.dependencytrack.model.AnalyzerIdentity; +import org.dependencytrack.model.Severity; +import org.dependencytrack.model.Vulnerability; +import org.dependencytrack.model.VulnerabilityAlias; +import org.dependencytrack.model.VulnerabilityAnalysisLevel; +import org.dependencytrack.model.mapping.PolicyProtoMapper; +import org.dependencytrack.notification.NotificationConstants; +import org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln; +import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.jdbi.NotificationSubjectDao; +import org.dependencytrack.policy.vulnerability.VulnerabilityPolicy; +import org.dependencytrack.policy.vulnerability.VulnerabilityPolicyEvaluator; +import org.dependencytrack.policy.vulnerability.VulnerabilityPolicyRating; +import org.dependencytrack.proto.notification.v1.Group; +import org.dependencytrack.proto.notification.v1.Level; +import org.dependencytrack.proto.notification.v1.Notification; +import org.dependencytrack.proto.notification.v1.Scope; +import org.dependencytrack.proto.vulnanalysis.v1.ScanKey; +import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; +import org.dependencytrack.proto.vulnanalysis.v1.ScanStatus; +import org.dependencytrack.proto.vulnanalysis.v1.Scanner; +import org.dependencytrack.proto.vulnanalysis.v1.ScannerResult; +import org.dependencytrack.util.PersistenceUtil; +import org.jdbi.v3.core.mapper.reflect.ColumnName; +import org.jdbi.v3.sqlobject.config.RegisterBeanMapper; +import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; +import org.jdbi.v3.sqlobject.customizer.BindBean; +import org.jdbi.v3.sqlobject.customizer.BindMethods; +import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; +import org.jdbi.v3.sqlobject.statement.SqlBatch; +import org.jdbi.v3.sqlobject.statement.SqlQuery; + +import javax.jdo.Query; +import javax.ws.rs.core.MultivaluedHashMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.ServiceLoader; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static java.util.Objects.requireNonNullElse; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.datanucleus.PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT; +import static org.datanucleus.PropertyNames.PROPERTY_RETAIN_VALUES; +import static org.dependencytrack.common.ConfigKey.VULNERABILITY_POLICY_ANALYSIS_ENABLED; +import static org.dependencytrack.parser.dependencytrack.ModelConverterCdxToVuln.convert; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.jdbi; +import static org.dependencytrack.proto.notification.v1.Group.GROUP_NEW_VULNERABILITY; +import static org.dependencytrack.proto.notification.v1.Group.GROUP_NEW_VULNERABLE_DEPENDENCY; +import static org.dependencytrack.proto.notification.v1.Level.LEVEL_INFORMATIONAL; +import static org.dependencytrack.proto.notification.v1.Scope.SCOPE_PORTFOLIO; +import static org.dependencytrack.proto.vulnanalysis.v1.ScanStatus.SCAN_STATUS_FAILED; +import static org.dependencytrack.proto.vulnanalysis.v1.Scanner.SCANNER_INTERNAL; +import static org.dependencytrack.util.NotificationUtil.generateNotificationContent; +import static org.dependencytrack.util.NotificationUtil.generateNotificationTitle; +import static org.dependencytrack.util.VulnerabilityUtil.canBeMirrored; +import static org.dependencytrack.util.VulnerabilityUtil.isAuthoritativeSource; +import static org.dependencytrack.util.VulnerabilityUtil.isMirroringEnabled; + +class VulnerabilityScanResultProcessor implements Processor { + + static final String PROCESSOR_NAME = "vuln.scan.result"; + + private static final Logger LOGGER = Logger.getLogger(VulnerabilityScanResultProcessor.class); + + private final KafkaEventDispatcher eventDispatcher; + private final VulnerabilityPolicyEvaluator vulnPolicyEvaluator; + private final ThreadLocal>> eventsToDispatch = ThreadLocal.withInitial(ArrayList::new); + + VulnerabilityScanResultProcessor() { + this(new KafkaEventDispatcher(), Config.getInstance().getPropertyAsBoolean(VULNERABILITY_POLICY_ANALYSIS_ENABLED) + ? ServiceLoader.load(VulnerabilityPolicyEvaluator.class).findFirst().orElseThrow() + : null); + } + + VulnerabilityScanResultProcessor(final KafkaEventDispatcher eventDispatcher, + final VulnerabilityPolicyEvaluator vulnPolicyEvaluator) { + this.eventDispatcher = eventDispatcher; + this.vulnPolicyEvaluator = vulnPolicyEvaluator; + } + + @Override + public void process(final ConsumerRecord record) throws ProcessingException { + try { + processInternal(record); + eventDispatcher.dispatchAllBlocking(eventsToDispatch.get()); + } finally { + eventsToDispatch.get().clear(); + } + } + + private void processInternal(final ConsumerRecord record) { + final ScanKey scanKey = record.key(); + final ScanResult scanResult = record.value(); + final UUID componentUuid = UUID.fromString(scanKey.getComponentUuid()); + final VulnerabilityAnalysisLevel analysisLevel = determineAnalysisLevel(record); + final boolean isNewComponent = determineIsComponentNew(record); + maybeQueueResultProcessedEvent(scanKey, scanResult); + + try (final var qm = new QueryManager()) { + // Do not unload fields upon commit (why is this even the default WTF). + qm.getPersistenceManager().setProperty(PROPERTY_RETAIN_VALUES, "true"); + qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); + + final Component component = jdbi(qm).withExtension(Dao.class, dao -> dao.getComponentByUuid(componentUuid)); + if (component == null) { + LOGGER.warn("Received result for component %s, but it does not exist (scanKey: %s)" + .formatted(componentUuid, prettyPrint(scanKey))); + return; + } + + for (final ScannerResult scannerResult : scanResult.getScannerResultsList()) { + processScannerResult(qm, component, scanKey, scannerResult, analysisLevel, isNewComponent); + } + } + } + + private void processScannerResult(final QueryManager qm, final Component component, + final ScanKey scanKey, final ScannerResult scannerResult, + final VulnerabilityAnalysisLevel analysisLevel, + final boolean isNewComponent) { + if (scannerResult.getStatus() == SCAN_STATUS_FAILED) { + final var message = "Scan of component %s with %s failed (scanKey: %s): %s" + .formatted(component.uuid(), scannerResult.getScanner(), prettyPrint(scanKey), scannerResult.getFailureReason()); + final var notification = Notification.newBuilder() + .setScope(Scope.SCOPE_SYSTEM) + .setGroup(Group.GROUP_ANALYZER) + .setLevel(Level.LEVEL_ERROR) + .setTimestamp(Timestamps.now()) + .setTitle(NotificationConstants.Title.ANALYZER_ERROR) + .setContent(message) + .build(); + eventsToDispatch.get().add(KafkaEventConverter.convert(component.projectUuid().toString(), notification)); + LOGGER.warn(message); + return; + } else if (scannerResult.getStatus() != ScanStatus.SCAN_STATUS_SUCCESSFUL) { + LOGGER.warn("Unable to process results from %s with status %s; Dropping record (scanKey: %s)" + .formatted(scannerResult.getScanner(), scannerResult.getStatus(), prettyPrint(scanKey))); + return; + } + + final Set syncedVulns = syncVulnerabilities(qm, scanKey, scannerResult); + LOGGER.debug("Synchronized %d vulnerabilities reported by %s for %s (scanKey: %s)" + .formatted(syncedVulns.size(), scannerResult.getScanner(), scanKey.getComponentUuid(), prettyPrint(scanKey))); + + final Map matchedPoliciesByVulnUuid = maybeEvaluateVulnPolicies(component, syncedVulns); + LOGGER.debug("Identified policy matches for %d/%d vulnerabilities (scanKey: %s)" + .formatted(matchedPoliciesByVulnUuid.size(), syncedVulns.size(), prettyPrint(scanKey))); + + final List newVulnUuids = synchronizeFindingsAndAnalyses(qm, component, syncedVulns, + scannerResult.getScanner(), matchedPoliciesByVulnUuid); + LOGGER.debug("Identified %d new vulnerabilities for %s with %s (scanKey: %s)" + .formatted(newVulnUuids.size(), scanKey.getComponentUuid(), scannerResult.getScanner(), prettyPrint(scanKey))); + + maybeQueueNotifications(qm, component, isNewComponent, analysisLevel, newVulnUuids); + } + + /** + * Synchronize vulnerabilities reported in a given {@link ScannerResult} with the datastore. + * + * @param qm The {@link QueryManager} to use + * @param scanKey The {@link ScanKey} associated with the {@link ScannerResult} + * @param scannerResult The {@link ScannerResult} to synchronize vulnerabilities from + * @return A {@link Set} of synchronized {@link Vulnerability}s + */ + private Set syncVulnerabilities(final QueryManager qm, final ScanKey scanKey, final ScannerResult scannerResult) { + final var syncedVulns = new HashSet(); + + for (final org.cyclonedx.proto.v1_4.Vulnerability reportedVuln : scannerResult.getBom().getVulnerabilitiesList()) { + final Vulnerability vuln; + try { + vuln = ModelConverterCdxToVuln.convert(qm, scannerResult.getBom(), reportedVuln, true); + } catch (RuntimeException e) { + LOGGER.error("Failed to convert vulnerability %s/%s (reported by %s for component %s) to internal model (scanKey: %s)" + .formatted(reportedVuln.getSource(), reportedVuln.getId(), scannerResult.getScanner(), scanKey.getComponentUuid(), prettyPrint(scanKey)), e); + continue; + } + + try { + final Vulnerability syncedVuln = syncVulnerability(qm, vuln, scannerResult.getScanner()); + + // Detach vulnerabilities from JDO persistence context. + // We do not want to trigger any DB interactions by accessing their fields later. + // Note that even PersistenceManager#detachCopy will load / unload fields based + // on the current FetchPlan. But we just want to keep the data we already have, + // and #makeTransientAll does exactly that. + qm.getPersistenceManager().makeTransient(syncedVuln); + + if (vuln.getAliases() != null && !vuln.getAliases().isEmpty()) { + final var syncedAliases = new ArrayList(); + for (VulnerabilityAlias alias : vuln.getAliases()) { + final VulnerabilityAlias syncedAlias = qm.synchronizeVulnerabilityAlias(alias); + qm.getPersistenceManager().makeTransient(syncedAlias); + syncedAliases.add(syncedAlias); + } + syncedVuln.setAliases(syncedAliases); + } + + syncedVulns.add(syncedVuln); + } catch (RuntimeException e) { + // Use a broad catch here, so we can still try to process other + // vulnerabilities, even though processing one of them failed. + + LOGGER.warn("Failed to synchronize vulnerability %s/%s (reported by %s for component %s; scanKey: %s)" + .formatted(vuln.getSource(), vuln.getVulnId(), scannerResult.getScanner(), scanKey.getComponentUuid(), prettyPrint(scanKey)), e); + } + } + + return syncedVulns; + } + + /** + * Synchronize a given {@link Vulnerability} as reported by a given {@link Scanner} with the datastore. + *

+ * This method differs from {@link QueryManager#synchronizeVulnerability(Vulnerability, boolean)} in that it expects + * an active {@link javax.jdo.Transaction}, and only calls setters of existing vulnerabilities when the respective + * value actually changed, saving network round-trips. + * + * @param qm The {@link QueryManager} to use + * @param vuln The {@link Vulnerability} to synchronize + * @param scanner The {@link AnalyzerIdentity} that reported the vulnerability + * @return The synchronized {@link Vulnerability} + * @throws IllegalStateException When no {@link javax.jdo.Transaction} is active + * @throws NoSuchElementException When the reported vulnerability is internal, but does not exist in the datastore + */ + private Vulnerability syncVulnerability(final QueryManager qm, final Vulnerability vuln, final Scanner scanner) { + // TODO: Refactor this to use JDBI instead. + // It is possible that the same vulnerability is reported for multiple components in parallel, + // causing unique constraint violations when attempting to INSERT into the VULNERABILITY table. + // In such cases, we can get away with simply retrying to SELECT or INSERT again. + return qm.runInRetryableTransaction(() -> { + final Vulnerability existingVuln; + final Query query = qm.getPersistenceManager().newQuery(Vulnerability.class); + try { + query.setFilter("vulnId == :vulnId && source == :source"); + query.setParameters(vuln.getVulnId(), vuln.getSource()); + existingVuln = query.executeUnique(); + } finally { + query.closeAll(); + } + + if (existingVuln == null) { + if (Vulnerability.Source.INTERNAL.name().equals(vuln.getSource())) { + throw new NoSuchElementException("An internal vulnerability with ID %s does not exist".formatted(vuln.getVulnId())); + } + + return qm.getPersistenceManager().makePersistent(vuln); + } + + if (canUpdateVulnerability(existingVuln, scanner)) { + final var differ = new PersistenceUtil.Differ<>(existingVuln, vuln); + + // TODO: Consider using something like javers to get a rich diff of WHAT changed; https://github.com/javers/javers + differ.applyIfChanged("title", Vulnerability::getTitle, existingVuln::setTitle); + differ.applyIfChanged("subTitle", Vulnerability::getSubTitle, existingVuln::setSubTitle); + differ.applyIfChanged("description", Vulnerability::getDescription, existingVuln::setDescription); + differ.applyIfChanged("detail", Vulnerability::getDetail, existingVuln::setDetail); + differ.applyIfChanged("recommendation", Vulnerability::getRecommendation, existingVuln::setRecommendation); + differ.applyIfChanged("references", Vulnerability::getReferences, existingVuln::setReferences); + differ.applyIfChanged("credits", Vulnerability::getCredits, existingVuln::setCredits); + differ.applyIfChanged("created", Vulnerability::getCreated, existingVuln::setCreated); + differ.applyIfChanged("published", Vulnerability::getPublished, existingVuln::setPublished); + differ.applyIfChanged("updated", Vulnerability::getUpdated, existingVuln::setUpdated); + differ.applyIfChanged("cwes", Vulnerability::getCwes, existingVuln::setCwes); + // Calling setSeverity nulls all CVSS and OWASP RR fields. getSeverity calculates the severity on-the-fly, + // and will return UNASSIGNED even when no severity is set explicitly. Thus, calling setSeverity + // must happen before CVSS and OWASP RR fields are set, to avoid null-ing them again. + differ.applyIfChanged("severity", Vulnerability::getSeverity, existingVuln::setSeverity); + differ.applyIfChanged("cvssV2BaseScore", Vulnerability::getCvssV2BaseScore, existingVuln::setCvssV2BaseScore); + differ.applyIfChanged("cvssV2ImpactSubScore", Vulnerability::getCvssV2ImpactSubScore, existingVuln::setCvssV2ImpactSubScore); + differ.applyIfChanged("cvssV2ExploitabilitySubScore", Vulnerability::getCvssV2ExploitabilitySubScore, existingVuln::setCvssV2ExploitabilitySubScore); + differ.applyIfChanged("cvssV2Vector", Vulnerability::getCvssV2Vector, existingVuln::setCvssV2Vector); + differ.applyIfChanged("cvssv3BaseScore", Vulnerability::getCvssV3BaseScore, existingVuln::setCvssV3BaseScore); + differ.applyIfChanged("cvssV3ImpactSubScore", Vulnerability::getCvssV3ImpactSubScore, existingVuln::setCvssV3ImpactSubScore); + differ.applyIfChanged("cvssV3ExploitabilitySubScore", Vulnerability::getCvssV3ExploitabilitySubScore, existingVuln::setCvssV3ExploitabilitySubScore); + differ.applyIfChanged("cvssV3Vector", Vulnerability::getCvssV3Vector, existingVuln::setCvssV3Vector); + differ.applyIfChanged("owaspRRLikelihoodScore", Vulnerability::getOwaspRRLikelihoodScore, existingVuln::setOwaspRRLikelihoodScore); + differ.applyIfChanged("owaspRRTechnicalImpactScore", Vulnerability::getOwaspRRTechnicalImpactScore, existingVuln::setOwaspRRTechnicalImpactScore); + differ.applyIfChanged("owaspRRBusinessImpactScore", Vulnerability::getOwaspRRBusinessImpactScore, existingVuln::setOwaspRRBusinessImpactScore); + differ.applyIfChanged("owaspRRVector", Vulnerability::getOwaspRRVector, existingVuln::setOwaspRRVector); + // Aliases of existingVuln will always be null, as they'd have to be fetched separately. + // Synchronization of aliases is performed after synchronizing the vulnerability. + // updated |= applyIfChanged(existingVuln, vuln, Vulnerability::getAliases, existingVuln::setAliases); + + differ.applyIfChanged("vulnerableVersions", Vulnerability::getVulnerableVersions, existingVuln::setVulnerableVersions); + differ.applyIfChanged("patchedVersions", Vulnerability::getPatchedVersions, existingVuln::setPatchedVersions); + // EPSS is an additional enrichment that no scanner currently provides. + // We don't want EPSS scores of CVEs to be purged just because the CVE information came from e.g. OSS Index. + differ.applyIfNonNullAndChanged("epssScore", Vulnerability::getEpssScore, existingVuln::setEpssScore); + differ.applyIfNonNullAndChanged("epssPercentile", Vulnerability::getEpssPercentile, existingVuln::setEpssPercentile); + + if (!differ.getDiffs().isEmpty()) { + // TODO: Send a notification? + // (But notifications should only be sent if the transaction was committed) + // TODO: Reduce to DEBUG; It's set to INFO for testing + LOGGER.info("Vulnerability %s/%s was updated by %s: %s".formatted(vuln.getSource(), vuln.getVulnId(), scanner, differ.getDiffs())); + } + } + + return existingVuln; + }, PersistenceUtil::isUniqueConstraintViolation); + } + + private Map maybeEvaluateVulnPolicies(final Component component, final Collection vulns) { + if (vulnPolicyEvaluator == null) { + return Collections.emptyMap(); + } + + final var policyProject = org.dependencytrack.proto.policy.v1.Project.newBuilder() + .setUuid(component.projectUuid().toString()) + .build(); + final var policyComponent = org.dependencytrack.proto.policy.v1.Component.newBuilder() + .setUuid(component.uuid().toString()) + .build(); + final List policyVulns = vulns.stream() + .map(PolicyProtoMapper::mapToProto) + .toList(); + + return vulnPolicyEvaluator.evaluate(policyVulns, policyComponent, policyProject); + } + + /** + * Associate a given {@link Collection} of {@link Vulnerability}s with a given {@link Component}, + * evaluate applicable {@link VulnerabilityPolicy}s, and apply the resulting analyses. + *

+ * If a {@link Vulnerability} was not previously associated with the {@link Component}, + * a {@link FindingAttribution} will be created for the {@link Scanner}. + * + * @param qm The {@link QueryManager} to use + * @param component The {@link Component} to associate with + * @param vulns The {@link Vulnerability}s to associate with + * @param scanner The {@link Scanner} that identified the association + * @param policiesByVulnUuid Matched {@link VulnerabilityPolicy}s grouped by {@link Vulnerability#getUuid()} + * @return A {@link List} of {@link Vulnerability}s, that were not previously associated with the {@link Component}, + * and which have not been suppressed via {@link VulnerabilityPolicy}. + */ + private List synchronizeFindingsAndAnalyses(final QueryManager qm, final Component component, + final Collection vulns, final Scanner scanner, + final Map policiesByVulnUuid) { + return jdbi(qm).inTransaction(jdbiHandle -> { + final var dao = jdbiHandle.attach(Dao.class); + + // Bulk-create new findings and corresponding scanner attributions. + final List newFindingVulnIds = dao.createFindings(component, vulns); + final List findingAttributions = newFindingVulnIds.stream() + .map(vulnId -> new FindingAttribution(vulnId, component.id(), component.projectId(), + convert(scanner).name(), UUID.randomUUID())) + .toList(); + dao.createFindingAttributions(findingAttributions); + + return maybeApplyPolicyAnalyses(dao, component, vulns, newFindingVulnIds, policiesByVulnUuid); + }); + } + + /** + * Apply analyses of matched {@link VulnerabilityPolicy}s. Do nothing when no policies matched. + * + * @param dao The {@link Dao} to use for persistence operations + * @param component The {@link Component} to apply analyses for + * @param vulns The {@link Vulnerability}s identified for the {@link Component} + * @param newFindingVulnIds IDs of {@link Vulnerability}s that newly affect the {@link Component} + * @param policiesByVulnUuid Matched {@link VulnerabilityPolicy}s grouped by {@link Vulnerability#getUuid()} + * @return A {@link List} of {@link Vulnerability}s, that were not previously associated with the {@link Component}, + * and which have not been suppressed via {@link VulnerabilityPolicy}. + */ + private List maybeApplyPolicyAnalyses(final Dao dao, final Component component, final Collection vulns, + final List newFindingVulnIds, final Map policiesByVulnUuid) { + // Unless we have any matching vulnerability policies, there's nothing to do! + if (policiesByVulnUuid.isEmpty()) { + return vulns.stream() + .filter(vuln -> newFindingVulnIds.contains(vuln.getId())) + .toList(); + } + + // Index vulnerabilities by ID and UUID for more efficient lookups. + final var vulnById = new HashMap(); + final var vulnByUuid = new HashMap(); + for (final Vulnerability vuln : vulns) { + vulnById.put(vuln.getId(), vuln); + vulnByUuid.put(vuln.getUuid(), vuln); + } + + // For all vulnerabilities with matching policies, bulk-fetch existing analyses. + // Index them by vulnerability UUID for more efficient access. + final Map existingAnalyses = dao.getAnalyses(component, policiesByVulnUuid.keySet()).stream() + .collect(Collectors.toMap(Analysis::getVulnUuid, Function.identity())); + + final var analysesToCreateOrUpdate = new ArrayList(); + final var analysisCommentsByVulnId = new MultivaluedHashMap(); + for (final Map.Entry vulnUuidAndPolicy : policiesByVulnUuid.entrySet()) { + final Vulnerability vuln = vulnByUuid.get(vulnUuidAndPolicy.getKey()); + final VulnerabilityPolicy policy = vulnUuidAndPolicy.getValue(); + final Analysis policyAnalysis; + try { + policyAnalysis = Analysis.fromPolicy(policy); + } catch (IllegalArgumentException e) { + LOGGER.warn("Unable to apply policy %s as it was found to be invalid".formatted(policy.name()), e); + continue; + } + + final Analysis existingAnalysis = existingAnalyses.get(vuln.getUuid()); + if (existingAnalysis == null) { + policyAnalysis.setComponentId(component.id()); + policyAnalysis.setProjectId(component.projectId()); + policyAnalysis.setVulnId(vuln.getId()); + policyAnalysis.setVulnUuid(vuln.getUuid()); + + // We'll create comments for analysisId=null for now, as the Analysis we're referring + // to hasn't been created yet. The analysisId is populated later, after bulk upserting + // all analyses. + final var commentFactory = new AnalysisCommentFactory(null, policy); + if (policyAnalysis.getState() != null) { + commentFactory.createComment("State: %s → %s" + .formatted(AnalysisState.NOT_SET, policyAnalysis.getState())); + } + if (policyAnalysis.getJustification() != null) { + commentFactory.createComment("Justification: %s → %s" + .formatted(AnalysisJustification.NOT_SET, policyAnalysis.getJustification())); + } + if (policyAnalysis.getResponse() != null) { + commentFactory.createComment("Response: %s → %s" + .formatted(AnalysisResponse.NOT_SET, policyAnalysis.response)); + } + if (policyAnalysis.getDetails() != null) { + commentFactory.createComment("Details: (None) → %s" + .formatted(policyAnalysis.details)); + } + if (policyAnalysis.getSuppressed()) { + commentFactory.createComment("Unsuppressed → Suppressed"); + } + if (policyAnalysis.getSeverity() != null) { + commentFactory.createComment("Severity: %s → %s" + .formatted(vuln.getSeverity(), policyAnalysis.getSeverity())); + } + if (policyAnalysis.getCvssV2Vector() != null) { + commentFactory.createComment("CVSSv2 Vector: %s → %s" + .formatted(requireNonNullElse(vuln.getCvssV2Vector(), "(None)"), policyAnalysis.getCvssV2Vector())); + } + if (policyAnalysis.getCvssV2Score() != null) { + commentFactory.createComment("CVSSv2 Score: %s → %s" + .formatted(requireNonNullElse(vuln.getCvssV2BaseScore(), "(None)"), policyAnalysis.getCvssV2Score())); + } + if (policyAnalysis.getCvssV3Vector() != null) { + commentFactory.createComment("CVSSv3 Vector: %s → %s" + .formatted(requireNonNullElse(vuln.getCvssV3Vector(), "(None)"), policyAnalysis.getCvssV3Vector())); + } + if (policyAnalysis.getCvssV3Score() != null) { + commentFactory.createComment("CVSSv3 Score: %s → %s" + .formatted(requireNonNullElse(vuln.getCvssV3BaseScore(), "(None)"), policyAnalysis.getCvssV3Score())); + } + if (policyAnalysis.getOwaspVector() != null) { + commentFactory.createComment("OWASP Vector: %s → %s" + .formatted(requireNonNullElse(vuln.getOwaspRRVector(), "(None)"), policyAnalysis.getOwaspVector())); + } + if (policyAnalysis.getOwaspScore() != null) { + commentFactory.createComment("OWASP Score: %s → %s" + .formatted(requireNonNullElse(vuln.getOwaspRRLikelihoodScore(), "(None)"), policyAnalysis.getOwaspScore())); + } + analysesToCreateOrUpdate.add(policyAnalysis); + analysisCommentsByVulnId.addAll(policyAnalysis.getVulnId(), commentFactory.getComments()); + } else { + boolean shouldUpdate = false; + final var commentFactory = new AnalysisCommentFactory(existingAnalysis.getId(), policy); + if (!Objects.equals(existingAnalysis.getState(), policyAnalysis.getState())) { + commentFactory.createComment("State: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getState(), AnalysisState.NOT_SET), + requireNonNullElse(policyAnalysis.getState(), AnalysisState.NOT_SET))); + + existingAnalysis.setState(policyAnalysis.getState()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getJustification(), policyAnalysis.getJustification())) { + commentFactory.createComment("Justification: %s → %s".formatted( + requireNonNullElse(existingAnalysis.justification, AnalysisJustification.NOT_SET), + requireNonNullElse(policyAnalysis.getJustification(), AnalysisJustification.NOT_SET))); + + existingAnalysis.setJustification(policyAnalysis.getJustification()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getResponse(), policyAnalysis.getResponse())) { + commentFactory.createComment("Response: %s → %s".formatted( + requireNonNullElse(existingAnalysis.response, AnalysisResponse.NOT_SET), + requireNonNullElse(policyAnalysis.getResponse(), AnalysisResponse.NOT_SET))); + + existingAnalysis.setResponse(policyAnalysis.getResponse()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.details, policyAnalysis.getDetails())) { + commentFactory.createComment("Details: %s → %s".formatted( + requireNonNullElse(existingAnalysis.details, "(None)"), + requireNonNullElse(policyAnalysis.getDetails(), "(None)"))); + + existingAnalysis.setDetails(policy.analysis().getDetails()); + shouldUpdate = true; + } + if (existingAnalysis.getSuppressed() == null || (existingAnalysis.getSuppressed() != policyAnalysis.getSuppressed())) { + final String previousState = existingAnalysis.suppressed ? "Suppressed" : "Unsuppressed"; + final String newState = policyAnalysis.getSuppressed() ? "Suppressed" : "Unsuppressed"; + commentFactory.createComment("%s → %s".formatted(previousState, newState)); + + existingAnalysis.setSuppressed(policy.analysis().isSuppress()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getSeverity(), policyAnalysis.getSeverity())) { + commentFactory.createComment("Severity: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getSeverity(), Severity.UNASSIGNED), + requireNonNullElse(policyAnalysis.getSeverity(), Severity.UNASSIGNED))); + + existingAnalysis.setSeverity(policyAnalysis.getSeverity()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getCvssV2Vector(), policyAnalysis.getCvssV2Vector())) { + commentFactory.createComment("CVSSv2 Vector: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getCvssV2Vector(), "(None)"), + requireNonNullElse(policyAnalysis.getCvssV2Vector(), "(None)"))); + + existingAnalysis.setCvssV2Vector(policyAnalysis.getCvssV2Vector()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getCvssV2Score(), policyAnalysis.getCvssV2Score())) { + commentFactory.createComment("CVSSv2 Score: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getCvssV2Score(), "(None)"), + requireNonNullElse(policyAnalysis.getCvssV2Score(), "(None)"))); + + existingAnalysis.setCvssV2Score(policyAnalysis.getCvssV2Score()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getCvssV3Vector(), policyAnalysis.getCvssV3Vector())) { + commentFactory.createComment("CVSSv3 Vector: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getCvssV3Vector(), "(None)"), + requireNonNullElse(policyAnalysis.getCvssV3Vector(), "(None)"))); + + existingAnalysis.setCvssV3Vector(policyAnalysis.getCvssV3Vector()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getCvssV3Score(), policyAnalysis.getCvssV3Score())) { + commentFactory.createComment("CVSSv3 Score: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getCvssV3Score(), "(None)"), + requireNonNullElse(policyAnalysis.getCvssV3Score(), "(None)"))); + + existingAnalysis.setCvssV3Score(policyAnalysis.getCvssV3Score()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getOwaspVector(), policyAnalysis.getOwaspVector())) { + commentFactory.createComment("OWASP Vector: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getOwaspVector(), "(None)"), + requireNonNullElse(policyAnalysis.getOwaspVector(), "(None)"))); + + existingAnalysis.setOwaspVector(policyAnalysis.getCvssV2Vector()); + shouldUpdate = true; + } + if (!Objects.equals(existingAnalysis.getOwaspScore(), policyAnalysis.getOwaspScore())) { + commentFactory.createComment("OWASP Score: %s → %s".formatted( + requireNonNullElse(existingAnalysis.getOwaspScore(), "(None)"), + requireNonNullElse(policyAnalysis.getOwaspScore(), "(None)"))); + + existingAnalysis.setOwaspScore(policyAnalysis.getOwaspScore()); + shouldUpdate = true; + } + if (shouldUpdate) { + analysesToCreateOrUpdate.add(existingAnalysis); + analysisCommentsByVulnId.addAll(existingAnalysis.getVulnId(), commentFactory.getComments()); + } + } + + // If the finding was suppressed, do not report it as new. + if (Boolean.TRUE.equals(policyAnalysis.getSuppressed())) { + newFindingVulnIds.remove(vuln.getId()); + } + } + + if (!analysesToCreateOrUpdate.isEmpty()) { + final List createdAnalyses = dao.createOrUpdateAnalyses(analysesToCreateOrUpdate); + // TODO: Construct notifications for PROJECT_AUDIT_CHANGE, but do not dispatch them here! + // They should be dispatched together with NEW_VULNERABILITY and NEW_VULNERABLE_DEPENDENCY + // notifications, AFTER this database transaction completed successfully. + + // Comments for new analyses do not have an analysis ID set yet, as that ID was not known prior + // to inserting the respective analysis record. Enrich comments with analysis IDs now that we know them. + for (final CreatedAnalysis createdAnalysis : createdAnalyses) { + analysisCommentsByVulnId.computeIfPresent(createdAnalysis.vulnId(), + (vulnId, comments) -> comments.stream() + .map(comment -> new AnalysisComment(createdAnalysis.id(), comment.comment(), comment.commenter())) + .toList()); + } + + dao.createAnalysisComments(analysisCommentsByVulnId.values().stream().flatMap(Collection::stream).toList()); + } + + return vulnById.entrySet().stream() + .filter(entry -> newFindingVulnIds.contains(entry.getKey())) + .map(Map.Entry::getValue) + .toList(); + } + + private void maybeQueueResultProcessedEvent(final ScanKey scanKey, final ScanResult scanResult) { + // Vulnerability scans targeting the entire portfolio are currently not tracked. + // There's no point in including results in the following repartition, and querying + // the database for their scan token, given the queries will never return anything anyway. + // Filtering results of portfolio analyses here also reduces the chance of hot partitions. + if (PortfolioVulnerabilityAnalysisEvent.CHAIN_IDENTIFIER.toString().equals(scanKey.getScanToken())) { + return; + } + + // Drop vulnerabilities from scanner results, as they can be rather large, and we don't need them anymore. + // Dropping them will save us some compression and network overhead during the repartition. + // We can remove this step should we ever need access to the vulnerabilities again. + final ScanResult strippedScanResult = scanResult.toBuilder() + .clearScannerResults() + .addAllScannerResults(scanResult.getScannerResultsList().stream() + .map(scannerResult -> scannerResult.toBuilder() + .clearBom() + .build()) + .toList()) + .build(); + + final var event = new KafkaEvent<>(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, scanKey.getScanToken(), strippedScanResult); + eventsToDispatch.get().add(event); + } + + /** + * Send {@link Group#GROUP_NEW_VULNERABLE_DEPENDENCY} and {@link Group#GROUP_NEW_VULNERABILITY} notifications + * for a given {@link Component}, if it was found to have at least one non-suppressed vulnerability. + * + * @param qm The {@link QueryManager} to use + * @param component The {@link Component} to send notifications for + * @param isNewComponent Whether {@code component} is new + * @param analysisLevel The {@link VulnerabilityAnalysisLevel} + * @param newVulns Newly identified {@link Vulnerability}s + */ + private void maybeQueueNotifications(final QueryManager qm, final Component component, final boolean isNewComponent, + final VulnerabilityAnalysisLevel analysisLevel, final List newVulns) { + if (newVulns.isEmpty()) { + return; + } + + final Timestamp notificationTimestamp = Timestamps.now(); + jdbi(qm).useExtension(NotificationSubjectDao.class, dao -> { + if (isNewComponent) { + dao.getForNewVulnerableDependency(component.uuid()) + .map(subject -> Notification.newBuilder() + .setScope(SCOPE_PORTFOLIO) + .setGroup(GROUP_NEW_VULNERABLE_DEPENDENCY) + .setLevel(LEVEL_INFORMATIONAL) + .setTimestamp(notificationTimestamp) + .setTitle(generateNotificationTitle(NotificationConstants.Title.NEW_VULNERABLE_DEPENDENCY, subject.getProject())) + .setContent(generateNotificationContent(subject.getComponent(), subject.getVulnerabilitiesList())) + .setSubject(Any.pack(subject)) + .build()) + .map(notification -> KafkaEventConverter.convert(component.projectUuid().toString(), notification)) + .ifPresent(eventsToDispatch.get()::add); + } + + dao.getForNewVulnerabilities(component.uuid(), newVulns.stream().map(Vulnerability::getUuid).toList(), analysisLevel).stream() + .map(subject -> Notification.newBuilder() + .setScope(SCOPE_PORTFOLIO) + .setGroup(GROUP_NEW_VULNERABILITY) + .setLevel(LEVEL_INFORMATIONAL) + .setTimestamp(notificationTimestamp) + .setTitle(generateNotificationTitle(NotificationConstants.Title.NEW_VULNERABILITY, subject.getProject())) + .setContent(generateNotificationContent(subject.getVulnerability())) + .setSubject(Any.pack(subject)) + .build()) + .map(notification -> KafkaEventConverter.convert(component.projectUuid().toString(), notification)) + .forEach(eventsToDispatch.get()::add); + }); + } + + private boolean canUpdateVulnerability(final Vulnerability vuln, final Scanner scanner) { + var canUpdate = true; + + // Results from the internal scanner only contain vulnId and source, nothing else. + // As they only refer to existing vulnerabilities in the database, no update must be performed. + canUpdate &= scanner != SCANNER_INTERNAL; + + // Internal vulnerabilities can only be updated via REST API. + canUpdate &= !Vulnerability.Source.INTERNAL.name().equals(vuln.getSource()); + + // If the scanner is also the authoritative source of the given vulnerability, + // it should be able to update it. This will be the case for the OSS Index scanner + // and sonatype-XXX vulnerabilities for example. + canUpdate &= isAuthoritativeSource(vuln, convert(scanner)) + // Alternatively, if the vulnerability could be mirrored, but mirroring + // is disabled, it is OK to override any existing data. + // + // Ideally, we'd track the data from all sources instead of just overriding + // it, but for now this will have to do it. + || (canBeMirrored(vuln) && !isMirroringEnabled(vuln)); + + return canUpdate; + } + + private static VulnerabilityAnalysisLevel determineAnalysisLevel(final ConsumerRecord record) { + return KafkaUtil.getEventHeader(record.headers(), KafkaEventHeaders.VULN_ANALYSIS_LEVEL) + .map(value -> { + try { + return VulnerabilityAnalysisLevel.valueOf(value); + } catch (IllegalArgumentException e) { + LOGGER.warn("The reported analysis type %s is invalid, assuming %s" + .formatted(value, VulnerabilityAnalysisLevel.PERIODIC_ANALYSIS)); + return VulnerabilityAnalysisLevel.PERIODIC_ANALYSIS; + } + }) + .orElse(VulnerabilityAnalysisLevel.PERIODIC_ANALYSIS); + } + + private static boolean determineIsComponentNew(final ConsumerRecord record) { + return KafkaUtil.getEventHeader(record.headers(), KafkaEventHeaders.IS_NEW_COMPONENT) + .map(Boolean::parseBoolean) + .orElse(false); + } + + private static String prettyPrint(final ScanKey scanKey) { + return "%s/%s".formatted(scanKey.getScanToken(), scanKey.getComponentUuid()); + } + + public interface Dao { + + @SqlQuery(""" + SELECT + "C"."ID" AS "id", + "C"."UUID" AS "uuid", + "P"."ID" AS "projectId", + "P"."UUID" AS "projectUuid" + FROM + "COMPONENT" AS "C" + INNER JOIN + "PROJECT" AS "P" ON "P"."ID" = "C"."PROJECT_ID" + WHERE + "C"."UUID" = (:uuid)::TEXT + """) + @RegisterConstructorMapper(Component.class) + Component getComponentByUuid(final UUID uuid); + + @SqlBatch(""" + INSERT INTO "COMPONENTS_VULNERABILITIES" + ("COMPONENT_ID", "VULNERABILITY_ID") + VALUES + (:component.id, :vuln.id) + ON CONFLICT DO NOTHING + RETURNING "VULNERABILITY_ID" + """) + @GetGeneratedKeys("VULNERABILITY_ID") + List createFindings(@BindMethods("component") final Component component, @BindBean("vuln") final Iterable vuln); + + @SqlBatch(""" + INSERT INTO "FINDINGATTRIBUTION" + ("VULNERABILITY_ID", "COMPONENT_ID", "PROJECT_ID", "ANALYZERIDENTITY", "ATTRIBUTED_ON", "UUID") + VALUES + (:vulnId, :componentId, :projectId, :analyzer, NOW(), (:uuid)::TEXT) + ON CONFLICT ("VULNERABILITY_ID", "COMPONENT_ID") DO NOTHING + """) + void createFindingAttributions(@BindMethods final Iterable attribution); + + @SqlQuery(""" + SELECT + "V"."ID" AS "vulnId", + "V"."UUID" AS "vulnUuid", + "A"."ID" AS "id", + "A"."COMPONENT_ID" AS "componentId", + "A"."PROJECT_ID" AS "projectId", + "A"."STATE" AS "state", + "A"."JUSTIFICATION" AS "justification", + "A"."RESPONSE" AS "response", + "A"."DETAILS" AS "details", + "A"."SUPPRESSED" AS "suppressed", + "A"."SEVERITY" AS "severity", + "A"."CVSSV2VECTOR" AS "cvssV2Vector", + "A"."CVSSV2SCORE" AS "cvssV2Score", + "A"."CVSSV3VECTOR" AS "cvssV3Vector", + "A"."CVSSV3SCORE" AS "cvssV3Score", + "A"."OWASPVECTOR" AS "owaspVector", + "A"."OWASPSCORE" AS "owaspScore" + FROM + "VULNERABILITY" AS "V" + INNER JOIN + "ANALYSIS" AS "A" ON "A"."VULNERABILITY_ID" = "V"."ID" + WHERE + "A"."COMPONENT_ID" = :component.id + AND "V"."UUID" = ANY((:vulnUuids)::TEXT[]) + """) + @RegisterBeanMapper(Analysis.class) + List getAnalyses(@BindMethods("component") final Component component, final Iterable vulnUuids); + + @SqlBatch(""" + INSERT INTO "ANALYSIS" + ("VULNERABILITY_ID", "COMPONENT_ID", "PROJECT_ID", "STATE", "JUSTIFICATION", "RESPONSE", "DETAILS", + "SUPPRESSED", "SEVERITY", "CVSSV2VECTOR", "CVSSV2SCORE", "CVSSV3VECTOR", "CVSSV3SCORE", "OWASPVECTOR", "OWASPSCORE") + VALUES + (:vulnId, :componentId, :projectId, :state, :justification, :response, :details, :suppressed, + :severity, :cvssV2Vector, :cvssV2Score, :cvssV3Vector, :cvssV3Score, :owaspVector, :owaspScore) + ON CONFLICT ("VULNERABILITY_ID", "COMPONENT_ID", "PROJECT_ID") DO UPDATE + SET + "STATE" = :state, + "JUSTIFICATION" = :justification, + "RESPONSE" = :response, + "DETAILS" = :details, + "SUPPRESSED" = :suppressed, + "SEVERITY" = :severity, + "CVSSV2VECTOR" = :cvssV2Vector, + "CVSSV2SCORE" = :cvssV2Score, + "CVSSV3VECTOR" = :cvssV3Vector, + "CVSSV3SCORE" = :cvssV3Score, + "OWASPVECTOR" = :owaspVector, + "OWASPSCORE" = :owaspScore + RETURNING "ID", "VULNERABILITY_ID" + """) + @GetGeneratedKeys({"ID", "VULNERABILITY_ID"}) + @RegisterConstructorMapper(CreatedAnalysis.class) + List createOrUpdateAnalyses(@BindBean final Iterable analysis); + + @SqlBatch(""" + INSERT INTO "ANALYSISCOMMENT" + ("ANALYSIS_ID", "TIMESTAMP", "COMMENT", "COMMENTER") + VALUES + (:analysisId, NOW(), :comment, :commenter) + """) + void createAnalysisComments(@BindMethods final Iterable comment); + + } + + public static class Analysis { + + private long id; + private long componentId; + private long projectId; + private long vulnId; + private UUID vulnUuid; + private AnalysisState state; + private AnalysisJustification justification; + private AnalysisResponse response; + private String details; + private Boolean suppressed; + private Severity severity; + private String cvssV2Vector; + private Double cvssV2Score; + private String cvssV3Vector; + private Double cvssV3Score; + private String owaspVector; + private Double owaspScore; + + private static Analysis fromPolicy(final VulnerabilityPolicy policy) { + final var analysis = new Analysis(); + if (policy.analysis().getState() != null) { + analysis.setState(switch (policy.analysis().getState()) { + case EXPLOITABLE -> AnalysisState.EXPLOITABLE; + case FALSE_POSITIVE -> AnalysisState.FALSE_POSITIVE; + case IN_TRIAGE -> AnalysisState.IN_TRIAGE; + case NOT_AFFECTED -> AnalysisState.NOT_AFFECTED; + case RESOLVED -> AnalysisState.RESOLVED; + }); + } else { + throw new IllegalArgumentException("Analysis of policy does not define a state"); + } + if (policy.analysis().getJustification() != null) { + analysis.setJustification(switch (policy.analysis().getJustification()) { + case CODE_NOT_PRESENT -> AnalysisJustification.CODE_NOT_PRESENT; + case CODE_NOT_REACHABLE -> AnalysisJustification.CODE_NOT_REACHABLE; + case PROTECTED_AT_PERIMETER -> AnalysisJustification.PROTECTED_AT_PERIMETER; + case PROTECTED_AT_RUNTIME -> AnalysisJustification.PROTECTED_AT_RUNTIME; + case PROTECTED_BY_COMPILER -> AnalysisJustification.PROTECTED_BY_COMPILER; + case PROTECTED_BY_MITIGATING_CONTROL -> AnalysisJustification.PROTECTED_BY_MITIGATING_CONTROL; + case REQUIRES_CONFIGURATION -> AnalysisJustification.REQUIRES_CONFIGURATION; + case REQUIRES_DEPENDENCY -> AnalysisJustification.REQUIRES_DEPENDENCY; + case REQUIRES_ENVIRONMENT -> AnalysisJustification.REQUIRES_ENVIRONMENT; + }); + } + if (policy.analysis().getVendorResponse() != null) { + analysis.setResponse(switch (policy.analysis().getVendorResponse()) { + case CAN_NOT_FIX -> AnalysisResponse.CAN_NOT_FIX; + case ROLLBACK -> AnalysisResponse.ROLLBACK; + case UPDATE -> AnalysisResponse.UPDATE; + case WILL_NOT_FIX -> AnalysisResponse.WILL_NOT_FIX; + case WORKAROUND_AVAILABLE -> AnalysisResponse.WORKAROUND_AVAILABLE; + }); + } + if (policy.analysis().getDetails() != null) { + analysis.setDetails(policy.analysis().getDetails()); + } + analysis.setSuppressed(policy.analysis().isSuppress()); + + if (policy.ratings() != null && !policy.ratings().isEmpty()) { + if (policy.ratings().size() > 3) { + throw new IllegalArgumentException("Policy defines more than three ratings"); + } + + final var methodsSeen = new HashSet(); + for (final VulnerabilityPolicyRating policyRating : policy.ratings()) { + if (policyRating.getMethod() == null) { + throw new IllegalArgumentException("Rating #%d does not define a method" + .formatted(policy.ratings().indexOf(policyRating))); + } + if (!methodsSeen.add(policyRating.getMethod())) { + throw new IllegalArgumentException("Rating method %s is defined more than once" + .formatted(policyRating.getMethod())); + } + if (policyRating.getSeverity() == null) { + throw new IllegalArgumentException("Rating #%d (%s) does not define a severity" + .formatted(policy.ratings().indexOf(policyRating), policyRating.getMethod())); + } + + analysis.setSeverity(switch (policyRating.getSeverity()) { + case INFO -> Severity.INFO; + case LOW -> Severity.LOW; + case MEDIUM -> Severity.MEDIUM; + case HIGH -> Severity.HIGH; + case CRITICAL -> Severity.CRITICAL; + }); + switch (policyRating.getMethod()) { + case CVSSV2 -> { + analysis.setCvssV2Vector(policyRating.getVector()); + analysis.setCvssV2Score(policyRating.getScore()); + } + case CVSSV3 -> { + analysis.setCvssV3Vector(policyRating.getVector()); + analysis.setCvssV3Score(policyRating.getScore()); + } + case OWASP -> { + analysis.setOwaspVector(policyRating.getVector()); + analysis.setOwaspScore(policyRating.getScore()); + } + } + } + } + + return analysis; + } + + public long getId() { + return id; + } + + public void setId(final long id) { + this.id = id; + } + + public long getComponentId() { + return componentId; + } + + public void setComponentId(final long componentId) { + this.componentId = componentId; + } + + public long getProjectId() { + return projectId; + } + + public void setProjectId(final long projectId) { + this.projectId = projectId; + } + + public long getVulnId() { + return vulnId; + } + + public void setVulnId(final long vulnId) { + this.vulnId = vulnId; + } + + public UUID getVulnUuid() { + return vulnUuid; + } + + public void setVulnUuid(final UUID vulnUuid) { + this.vulnUuid = vulnUuid; + } + + public AnalysisState getState() { + return state; + } + + public void setState(final AnalysisState state) { + this.state = state; + } + + public AnalysisJustification getJustification() { + return justification; + } + + public void setJustification(final AnalysisJustification justification) { + this.justification = justification; + } + + public AnalysisResponse getResponse() { + return response; + } + + public void setResponse(final AnalysisResponse response) { + this.response = response; + } + + public String getDetails() { + return details; + } + + public void setDetails(final String details) { + this.details = details; + } + + public Boolean getSuppressed() { + return suppressed; + } + + public void setSuppressed(final Boolean suppressed) { + this.suppressed = suppressed; + } + + public Severity getSeverity() { + return severity; + } + + public void setSeverity(final Severity severity) { + this.severity = severity; + } + + public String getCvssV2Vector() { + return cvssV2Vector; + } + + public void setCvssV2Vector(final String cvssV2Vector) { + this.cvssV2Vector = cvssV2Vector; + } + + public Double getCvssV2Score() { + return cvssV2Score; + } + + public void setCvssV2Score(final Double cvssV2Score) { + this.cvssV2Score = cvssV2Score; + } + + public String getCvssV3Vector() { + return cvssV3Vector; + } + + public void setCvssV3Vector(final String cvssV3Vector) { + this.cvssV3Vector = cvssV3Vector; + } + + public Double getCvssV3Score() { + return cvssV3Score; + } + + public void setCvssV3Score(final Double cvssV3Score) { + this.cvssV3Score = cvssV3Score; + } + + public String getOwaspVector() { + return owaspVector; + } + + public void setOwaspVector(final String owaspVector) { + this.owaspVector = owaspVector; + } + + public Double getOwaspScore() { + return owaspScore; + } + + public void setOwaspScore(final Double owaspScore) { + this.owaspScore = owaspScore; + } + + } + + public record CreatedAnalysis(long id, @ColumnName("VULNERABILITY_ID") long vulnId) { + } + + public record AnalysisComment(Long analysisId, String comment, String commenter) { + } + + private static final class AnalysisCommentFactory { + + private final Long analysisId; + private final VulnerabilityPolicy policy; + private final String commenter; + private final List comments; + + private AnalysisCommentFactory(final Long analysisId, VulnerabilityPolicy policy) { + this.analysisId = analysisId; + this.policy = policy; + this.commenter = createCommenter(policy); + this.comments = new ArrayList<>(); + } + + private void createComment(final String comment) { + comments.add(new AnalysisComment(this.analysisId, comment, this.commenter)); + } + + private List getComments() { + if (comments.isEmpty()) { + return comments; + } + + // If we have comments already, additionally include what the policy matched on. + // Include this as the very first comment, and do not modify the original list. + final var commentsCopy = new ArrayList(); + commentsCopy.add(new AnalysisComment(this.analysisId, "Matched on condition(s):\n%s" + .formatted(policy.conditions().stream().map("- %s"::formatted).collect(Collectors.joining("\n"))), this.commenter)); + commentsCopy.addAll(comments); + return commentsCopy; + } + + private static String createCommenter(final VulnerabilityPolicy policy) { + if (isNotBlank(policy.author())) { + return "[Policy{Name=%s, Author=%s}]".formatted(policy.name(), policy.author()); + } + + return "[Policy{Name=%s}]".formatted(policy.name()); + } + + } + + public record Component(long id, UUID uuid, long projectId, UUID projectUuid) { + } + + public record FindingAttribution(long vulnId, long componentId, long projectId, String analyzer, UUID uuid) { + } + +} diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java index 44db29226..113f1dcbc 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/AbstractProcessorTest.java @@ -1,6 +1,7 @@ package org.dependencytrack.event.kafka.processor; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeaders; import org.apache.kafka.common.record.TimestampType; import org.dependencytrack.AbstractPostgresEnabledTest; @@ -8,15 +9,51 @@ import java.time.Instant; import java.util.Optional; +import static java.util.Objects.requireNonNullElseGet; + abstract class AbstractProcessorTest extends AbstractPostgresEnabledTest { - ConsumerRecord createConsumerRecord(final K key, final V value) { - return createConsumerRecord(key, value, Instant.now()); + static ConsumerRecordBuilder aConsumerRecord(final K key, final V value) { + return new ConsumerRecordBuilder<>(key, value); } - ConsumerRecord createConsumerRecord(final K key, final V value, final Instant timestamp) { - return new ConsumerRecord<>("topic", 0, 1, timestamp.toEpochMilli(), TimestampType.CREATE_TIME, -1, -1, - key, value, new RecordHeaders(), Optional.empty()); + static final class ConsumerRecordBuilder { + + private final K key; + private final V value; + private Instant timestamp; + private Headers headers; + + private ConsumerRecordBuilder(final K key, final V value) { + this.key = key; + this.value = value; + } + + ConsumerRecordBuilder withTimestamp(final Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + ConsumerRecordBuilder withHeaders(final Headers headers) { + this.headers = headers; + return this; + } + + ConsumerRecord build() { + final Instant timestamp = requireNonNullElseGet(this.timestamp, Instant::now); + final Headers headers = requireNonNullElseGet(this.headers, RecordHeaders::new); + return new ConsumerRecord<>( + "topicName", + /* partition */ 0, + /* offset */ 1, + timestamp.toEpochMilli(), TimestampType.CREATE_TIME, + /* serializedKeySize */ -1, + /* serializedValueSize */ -1, + this.key, this.value, + headers, + /* leaderEpoch */ Optional.empty()); + } + } } diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java index 4dcde9801..93c035e52 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/MirroredVulnerabilityProcessorTest.java @@ -65,7 +65,7 @@ public void testProcessNvdVuln() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); @@ -200,7 +200,7 @@ public void testProcessGitHubVuln() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("GITHUB/GHSA-fxwm-579q-49qq", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("GITHUB/GHSA-fxwm-579q-49qq", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-fxwm-579q-49qq"); assertThat(vuln).isNotNull(); @@ -407,7 +407,7 @@ public void testProcessOsvVuln() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("OSV/GHSA-2cc5-23r7-vc4v", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("OSV/GHSA-2cc5-23r7-vc4v", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("GITHUB", "GHSA-2cc5-23r7-vc4v"); assertThat(vuln).isNotNull(); @@ -556,7 +556,7 @@ public void testProcessVulnWithoutAffects() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); @@ -625,7 +625,7 @@ public void testProcessVulnWithUnmatchedAffectsBomRef() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); @@ -720,7 +720,7 @@ public void testProcessVulnWithVersConstraints() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); @@ -989,7 +989,7 @@ public void testProcessVulnWithInvalidCpeOrPurl() throws Exception { """; final var processor = new MirroredVulnerabilityProcessor(); - processor.process(createConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson))); + processor.process(aConsumerRecord("NVD/CVE-2022-40489", generateBomFromJson(bovJson)).build()); final Vulnerability vuln = qm.getVulnerabilityByVulnId("NVD", "CVE-2022-40489"); assertThat(vuln).isNotNull(); diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java index e763c8546..f4c953870 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java @@ -49,7 +49,7 @@ public void processNewMetaModelTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); final RepositoryMetaComponent metaComponent = qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); assertThat(metaComponent).isNotNull(); @@ -69,7 +69,7 @@ public void processWithoutComponentDetailsTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); final Query query = qm.getPersistenceManager().newQuery(RepositoryMetaComponent.class); query.setResult("count(this)"); @@ -99,7 +99,7 @@ public void processUpdateExistingMetaModelTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); qm.getPersistenceManager().refresh(metaComponent); assertThat(metaComponent).isNotNull(); @@ -135,7 +135,7 @@ public void processUpdateOutOfOrderMetaModelTest() throws Exception { // Pipe in a record that was produced 10 seconds ago, 5 seconds before metaComponent's lastCheck. final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result, Instant.now().minusSeconds(10))); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).withTimestamp(Instant.now().minusSeconds(10)).build()); qm.getPersistenceManager().refresh(metaComponent); assertThat(metaComponent).isNotNull(); @@ -181,7 +181,7 @@ public void processUpdateIntegrityResultTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -234,7 +234,7 @@ public void testIntegrityCheckWhenComponentHashIsMissing() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -285,7 +285,7 @@ public void testIntegrityAnalysisWillNotBePerformedIfNoIntegrityDataInResult() t .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNull(); @@ -324,7 +324,7 @@ public void testIntegrityCheckWillNotBeDoneIfComponentUuidAndIntegrityDataIsMiss .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNull(); @@ -364,7 +364,7 @@ public void testIntegrityIfResultHasIntegrityDataAndComponentUuidIsMissing() thr .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNotNull(); @@ -394,7 +394,7 @@ public void testIntegrityCheckWillNotBeDoneIfComponentIsNotInDb() throws Excepti .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(uuid); assertThat(analysis).isNull(); @@ -432,7 +432,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissing() throws Exceptio .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -481,7 +481,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -532,7 +532,7 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -567,7 +567,7 @@ public void processUpdateIntegrityResultNotAvailableTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -598,7 +598,7 @@ public void processUpdateOldIntegrityResultSent() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); @@ -632,7 +632,7 @@ public void processBothMetaModelAndIntegrityMeta() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); final RepositoryMetaComponent metaComponent = qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); @@ -667,7 +667,7 @@ public void processUpdateIntegrityResultNotSentTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(createConsumerRecord("pkg:maven/foo/bar@1.2.3", result)); + processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); qm.getPersistenceManager().refresh(integrityMetaComponent); integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); diff --git a/src/test/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessorTest.java similarity index 89% rename from src/test/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessorTest.java rename to src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessorTest.java index 364407e13..e6a30780e 100644 --- a/src/test/java/org/dependencytrack/event/kafka/streams/processor/VulnerabilityScanResultProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/VulnerabilityScanResultProcessorTest.java @@ -1,30 +1,20 @@ -package org.dependencytrack.event.kafka.streams.processor; +package org.dependencytrack.event.kafka.processor; import com.google.protobuf.Timestamp; import junitparams.JUnitParamsRunner; import junitparams.Parameters; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeaders; -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.TestInputTopic; -import org.apache.kafka.streams.TestOutputTopic; -import org.apache.kafka.streams.TopologyTestDriver; -import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.Produced; -import org.apache.kafka.streams.test.TestRecord; import org.cyclonedx.proto.v1_4.Advisory; import org.cyclonedx.proto.v1_4.Bom; import org.cyclonedx.proto.v1_4.Property; import org.cyclonedx.proto.v1_4.Source; import org.cyclonedx.proto.v1_4.VulnerabilityRating; import org.cyclonedx.proto.v1_4.VulnerabilityReference; -import org.dependencytrack.AbstractPostgresEnabledTest; import org.dependencytrack.TestCacheManager; +import org.dependencytrack.event.kafka.KafkaEventDispatcher; import org.dependencytrack.event.kafka.KafkaEventHeaders; import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufDeserializer; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerde; -import org.dependencytrack.event.kafka.serialization.KafkaProtobufSerializer; import org.dependencytrack.model.Analysis; import org.dependencytrack.model.AnalysisComment; import org.dependencytrack.model.AnalysisJustification; @@ -56,7 +46,6 @@ import org.dependencytrack.proto.vulnanalysis.v1.ScanResult; import org.dependencytrack.proto.vulnanalysis.v1.Scanner; import org.dependencytrack.proto.vulnanalysis.v1.ScannerResult; -import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -88,11 +77,9 @@ import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; @RunWith(JUnitParamsRunner.class) -public class VulnerabilityScanResultProcessorTest extends AbstractPostgresEnabledTest { +public class VulnerabilityScanResultProcessorTest extends AbstractProcessorTest { - private TopologyTestDriver testDriver; - private TestInputTopic inputTopic; - private TestOutputTopic outputTopic; + private VulnerabilityScanResultProcessor processor; @Before public void setUp() throws Exception { @@ -102,35 +89,13 @@ public void setUp() throws Exception { final var scriptHost = new CelPolicyScriptHost(cacheManager, CelPolicyType.VULNERABILITY); final var policyProvider = new DatabaseVulnerabilityPolicyProvider(); final var policyEvaluator = new CelVulnerabilityPolicyEvaluator(policyProvider, scriptHost, cacheManager); - - final var streamsBuilder = new StreamsBuilder(); - streamsBuilder - .stream("input-topic", Consumed - .with(new KafkaProtobufSerde<>(ScanKey.parser()), new KafkaProtobufSerde<>(ScanResult.parser()))) - .processValues(() -> new VulnerabilityScanResultProcessor(policyEvaluator)) - .to("output-topic", Produced - .with(new KafkaProtobufSerde<>(ScanKey.parser()), new KafkaProtobufSerde<>(ScanResult.parser()))); - - testDriver = new TopologyTestDriver(streamsBuilder.build()); - inputTopic = testDriver.createInputTopic("input-topic", - new KafkaProtobufSerializer<>(), new KafkaProtobufSerializer<>()); - outputTopic = testDriver.createOutputTopic("output-topic", - new KafkaProtobufDeserializer<>(ScanKey.parser()), new KafkaProtobufDeserializer<>(ScanResult.parser())); + processor = new VulnerabilityScanResultProcessor(new KafkaEventDispatcher(), policyEvaluator); new CweImporter().processCweDefinitions(); // Required for CWE mapping } - @After - public void tearDown() { - if (testDriver != null) { - testDriver.close(); - } - - super.tearDown(); - } - @Test - public void dropFailedScanResultTest() { + public void dropFailedScanResultTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -152,11 +117,14 @@ public void dropFailedScanResultTest() { .setFailureReason("just because")) .build(); - inputTopic.pipeInput(scanKey, scanResult); - - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_ANALYZER.name()); final Notification notification = deserializeValue(KafkaTopics.NOTIFICATION_ANALYZER, record); @@ -172,7 +140,7 @@ record -> { } @Test - public void dropPendingScanResultTest() { + public void dropPendingScanResultTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -193,15 +161,17 @@ public void dropPendingScanResultTest() { .setStatus(SCAN_STATUS_PENDING)) .build(); - inputTopic.pipeInput(scanKey, scanResult); - - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); - assertThat(kafkaMockProducer.history()).isEmpty(); + assertThat(kafkaMockProducer.history()).satisfiesExactly(record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }); } @Test - public void processSuccessfulScanResultWhenComponentDoesNotExistTest() { + public void processSuccessfulScanResultWhenComponentDoesNotExistTest() throws Exception { final var componentUuid = UUID.randomUUID(); final var scanToken = UUID.randomUUID().toString(); final var scanKey = ScanKey.newBuilder().setScanToken(scanToken).setComponentUuid(componentUuid.toString()).build(); @@ -216,15 +186,17 @@ public void processSuccessfulScanResultWhenComponentDoesNotExistTest() { .setBom(Bom.newBuilder().addVulnerabilities(createVuln("INT-001", "INTERNAL")))) .build(); - inputTopic.pipeInput(scanKey, scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); - - assertThat(kafkaMockProducer.history()).isEmpty(); + assertThat(kafkaMockProducer.history()).satisfiesExactly(record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }); } @Test - public void processSuccessfulScanResult() { + public void processSuccessfulScanResult() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -264,9 +236,7 @@ public void processSuccessfulScanResult() { headers.add(KafkaEventHeaders.VULN_ANALYSIS_LEVEL, VulnerabilityAnalysisLevel.BOM_UPLOAD_ANALYSIS.name().getBytes()); headers.add(KafkaEventHeaders.IS_NEW_COMPONENT, "true".getBytes()); - inputTopic.pipeInput(new TestRecord<>(scanKey, scanResult, headers)); - - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).withHeaders(headers).build()); qm.getPersistenceManager().refresh(component); assertThat(component.getVulnerabilities()).satisfiesExactlyInAnyOrder( @@ -295,6 +265,11 @@ public void processSuccessfulScanResult() { ); assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_NEW_VULNERABLE_DEPENDENCY.name()); final Notification notification = deserializeValue(KafkaTopics.NOTIFICATION_NEW_VULNERABLE_DEPENDENCY, record); @@ -334,7 +309,7 @@ record -> { } @Test - public void processSuccessfulScanResultWithExistingFindingTest() { + public void processSuccessfulScanResultWithExistingFindingTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -362,7 +337,7 @@ public void processSuccessfulScanResultWithExistingFindingTest() { .setBom(Bom.newBuilder().addVulnerabilities(createVuln("CVE-001", "NVD")))) .build(); - inputTopic.pipeInput(scanKey, scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().refreshAll(component, vulnerability); assertThat(component.getVulnerabilities()).satisfiesExactly( @@ -378,7 +353,11 @@ public void processSuccessfulScanResultWithExistingFindingTest() { assertThat(attribution.getAnalyzerIdentity()).isEqualTo(AnalyzerIdentity.OSSINDEX_ANALYZER); // Because the vulnerability was reported already, no notification must be sent. - assertThat(kafkaMockProducer.history()).isEmpty(); + assertThat(kafkaMockProducer.history()).satisfiesExactly(record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }); } private Object[] canUpdateExistingVulnerabilityTestParams() { @@ -422,7 +401,7 @@ private Object[] canUpdateExistingVulnerabilityTestParams() { @Parameters(method = "canUpdateExistingVulnerabilityTestParams") public void canUpdateExistingVulnerabilityTest(final String vulnId, final String vulnSource, final Scanner scanner, final ConfigPropertyConstants mirrorSourceConfigProperty, - final String mirrorSourceConfigPropertyValue, final boolean expectModified) { + final String mirrorSourceConfigPropertyValue, final boolean expectModified) throws Exception { if (mirrorSourceConfigProperty != null && mirrorSourceConfigPropertyValue != null) { qm.createConfigProperty( mirrorSourceConfigProperty.getGroupName(), @@ -462,7 +441,7 @@ public void canUpdateExistingVulnerabilityTest(final String vulnId, final String .build()))) .build(); - inputTopic.pipeInput(scanKey, scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().refreshAll(component, vulnerability); assertThat(component.getVulnerabilities()).satisfiesExactly( @@ -479,7 +458,7 @@ public void canUpdateExistingVulnerabilityTest(final String vulnId, final String } @Test - public void updateExistingVulnerabilityTest() { + public void updateExistingVulnerabilityTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -564,7 +543,7 @@ public void updateExistingVulnerabilityTest() { .build()))) .build(); - inputTopic.pipeInput(scanKey, scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().refreshAll(component, vulnerability); assertThat(component.getVulnerabilities()).hasSize(1); @@ -601,7 +580,7 @@ public void updateExistingVulnerabilityTest() { } @Test - public void analysisThroughPolicyNewAnalysisTest() { + public void analysisThroughPolicyNewAnalysisTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -651,8 +630,7 @@ public void analysisThroughPolicyNewAnalysisTest() { createVuln(newVuln.getVulnId(), newVuln.getSource()) )))) .build(); - inputTopic.pipeInput(new TestRecord<>(scanKey, scanResult)); - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().evictAll(); assertThat(component.getVulnerabilities()).satisfiesExactly( @@ -691,6 +669,11 @@ public void analysisThroughPolicyNewAnalysisTest() { // TODO: There should be PROJECT_AUDIT_CHANGE notifications. assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }, record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_NEW_VULNERABILITY.name()); final Notification notification = deserializeValue(KafkaTopics.NOTIFICATION_NEW_VULNERABILITY, record); @@ -708,7 +691,7 @@ record -> { } @Test - public void analysisThroughPolicyNewAnalysisSuppressionTest() { + public void analysisThroughPolicyNewAnalysisSuppressionTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -750,8 +733,7 @@ public void analysisThroughPolicyNewAnalysisSuppressionTest() { createVuln(newVuln.getVulnId(), newVuln.getSource()) )))) .build(); - inputTopic.pipeInput(new TestRecord<>(scanKey, scanResult)); - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().evictAll(); assertThat(component.getVulnerabilities()).satisfiesExactly( @@ -785,11 +767,15 @@ public void analysisThroughPolicyNewAnalysisSuppressionTest() { // The vulnerability was suppressed, so no notifications to be expected. // TODO: There should be PROJECT_AUDIT_CHANGE notifications. - assertThat(kafkaMockProducer.history()).isEmpty(); + assertThat(kafkaMockProducer.history()).satisfiesExactly(record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }); } @Test - public void analysisThroughPolicyExistingDifferentAnalysisTest() { + public void analysisThroughPolicyExistingDifferentAnalysisTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -851,8 +837,7 @@ public void analysisThroughPolicyExistingDifferentAnalysisTest() { createVuln(vuln.getVulnId(), vuln.getSource()) )))) .build(); - inputTopic.pipeInput(new TestRecord<>(scanKey, scanResult)); - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().evictAll(); assertThat(component.getVulnerabilities()).satisfiesExactly( @@ -896,11 +881,15 @@ public void analysisThroughPolicyExistingDifferentAnalysisTest() { // The vulnerability already existed, so no notifications to be expected. // TODO: There should be PROJECT_AUDIT_CHANGE notifications. - assertThat(kafkaMockProducer.history()).isEmpty(); + assertThat(kafkaMockProducer.history()).satisfiesExactly(record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }); } @Test - public void analysisThroughPolicyExistingEqualAnalysisTest() { + public void analysisThroughPolicyExistingEqualAnalysisTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -957,8 +946,7 @@ public void analysisThroughPolicyExistingEqualAnalysisTest() { createVuln(vuln.getVulnId(), vuln.getSource()) )))) .build(); - inputTopic.pipeInput(new TestRecord<>(scanKey, scanResult)); - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().evictAll(); assertThat(component.getVulnerabilities()).satisfiesExactly( @@ -984,11 +972,15 @@ public void analysisThroughPolicyExistingEqualAnalysisTest() { }); // The vulnerability already existed, so no notifications to be expected. - assertThat(kafkaMockProducer.history()).isEmpty(); + assertThat(kafkaMockProducer.history()).satisfiesExactly(record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }); } @Test - public void analysisThroughPolicyWithAliasesTest() { + public void analysisThroughPolicyWithAliasesTest() throws Exception { final var project = new Project(); project.setName("acme-app"); project.setVersion("1.0.0"); @@ -1061,8 +1053,7 @@ public void analysisThroughPolicyWithAliasesTest() { .build() )))) .build(); - inputTopic.pipeInput(new TestRecord<>(scanKey, scanResult)); - assertThat(outputTopic.readValuesToList()).containsOnly(scanResult); + processor.process(aConsumerRecord(scanKey, scanResult).build()); qm.getPersistenceManager().evictAll(); assertThat(component.getVulnerabilities()).satisfiesExactlyInAnyOrder( @@ -1085,6 +1076,23 @@ public void analysisThroughPolicyWithAliasesTest() { assertThat(qm.getAnalysis(component, v)).isNull(); } ); + + assertThat(kafkaMockProducer.history()).satisfiesExactly( + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED.name()); + final ScanResult strippedResult = deserializeValue(KafkaTopics.VULN_ANALYSIS_RESULT_PROCESSED, record); + assertThat(strippedResult.getScannerResultsList()).noneMatch(ScannerResult::hasBom); + }, + record -> { + assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_NEW_VULNERABILITY.name()); + final Notification notification = deserializeValue(KafkaTopics.NOTIFICATION_NEW_VULNERABILITY, record); + assertThat(notification.getScope()).isEqualTo(SCOPE_PORTFOLIO); + assertThat(notification.getLevel()).isEqualTo(LEVEL_INFORMATIONAL); + assertThat(notification.getGroup()).isEqualTo(GROUP_NEW_VULNERABILITY); + assertThat(notification.getSubject().is(NewVulnerabilitySubject.class)).isTrue(); + final var subject = notification.getSubject().unpack(NewVulnerabilitySubject.class); + assertThat(subject.getVulnerabilityAnalysisLevel()).isEqualTo("PERIODIC_ANALYSIS"); + }); } private org.cyclonedx.proto.v1_4.Vulnerability createVuln(final String id, final String source) {