Set subscribe pattern to wildcard prefix

caraml-dev · Dec 19, 2024 · 287758b · 287758b
1 parent 190eec2
commit 287758b
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/caraml-store-spark/src/main/scala/dev/caraml/spark/StreamingPipeline.scala b/caraml-store-spark/src/main/scala/dev/caraml/spark/StreamingPipeline.scala
@@ -54,7 +54,8 @@ object StreamingPipeline extends BasePipeline with Serializable {
         sparkSession.readStream
           .format("kafka")
           .option("kafka.bootstrap.servers", source.bootstrapServers)
-          .option("subscribe", source.topic)
+//          .option("subscribe", source.topic)
+          .option("subscribePattern", s".*\\.${source.topic}")
           .load()
       case source: MemoryStreamingSource =>
         source.read