[receiver/kafkareceiver] add topic_regex configuration option

.chloggen/feat_kafkareceiver-regex-topics.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: 'kafkareceiver'
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add 'topic_regex' configuration option to kafkareceiver. It allows to subscribe to topics based on name pattern."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36908]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/kafkareceiver/README.md

@@ -28,7 +28,8 @@ The following settings can be optionally configured:
 - `brokers` (default = localhost:9092): The list of kafka brokers
 - `resolve_canonical_bootstrap_servers_only` (default = false): Whether to resolve then reverse-lookup broker IPs during startup
 - `topic` (default = otlp_spans for traces, otlp_metrics for metrics, otlp_logs for logs): The name of the kafka topic to read from.
-  Only one telemetry type may be used for a given topic.
+  Only one telemetry type may be used for a given topic. Only one setting option `topic` or `topic_regex` can be used.
+- `topic_regex` (no default): Used for declaring topics subscription as name pattern. Only one setting option `topic` or `topic_regex` can be used.
 - `encoding` (default = otlp_proto): The encoding of the payload received from kafka. Supports encoding extensions. Tries to load an encoding extension and falls back to internal encodings if no extension was loaded. Available internal encodings:
   - `otlp_proto`: the payload is deserialized to `ExportTraceServiceRequest`, `ExportLogsServiceRequest` or `ExportMetricsServiceRequest` respectively.
   - `otlp_json`: the payload is deserialized to `ExportTraceServiceRequest` `ExportLogsServiceRequest` or `ExportMetricsServiceRequest` respectively using JSON encoding.

receiver/kafkareceiver/config.go

@@ -4,6 +4,7 @@
 package kafkareceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver"
 
 import (
+	"fmt"
 	"time"
 
 	"go.opentelemetry.io/collector/component"
@@ -52,8 +53,10 @@ type Config struct {
 	SessionTimeout time.Duration `mapstructure:"session_timeout"`
 	// Heartbeat interval for the Kafka consumer
 	HeartbeatInterval time.Duration `mapstructure:"heartbeat_interval"`
-	// The name of the kafka topic to consume from (default "otlp_spans" for traces, "otlp_metrics" for metrics, "otlp_logs" for logs)
+	// The name of the kafka topic to consume from (default "otlp_spans" for traces, "otlp_metrics" for metrics, "otlp_logs" for logs). If topics_regex is used, this field must remain empty.
 	Topic string `mapstructure:"topic"`
+	// Name pattern of the kafka topics to consume from. If topic is used, this field must remain empty.
+	TopicRegex string `mapstructure:"topic_regex"`
 	// Encoding of the messages (default "otlp_proto")
 	Encoding string `mapstructure:"encoding"`
 	// The consumer group that receiver will be consuming messages from (default "otel-collector")
@@ -96,5 +99,8 @@ var _ component.Config = (*Config)(nil)
 
 // Validate checks the receiver configuration is valid
 func (cfg *Config) Validate() error {
+	if len(cfg.Topic) > 0 && len(cfg.TopicRegex) > 0 {
+		return fmt.Errorf("only one setting 'topic' or 'topics_regex' can be used")
+	}
 	return nil
 }

receiver/kafkareceiver/config_test.go

@@ -34,6 +34,7 @@ func TestLoadConfig(t *testing.T) {
 			id: component.NewIDWithName(metadata.Type, ""),
 			expected: &Config{
 				Topic:                                "spans",
+				TopicRegex:                           "",
 				Encoding:                             "otlp_proto",
 				Brokers:                              []string{"foo:123", "bar:456"},
 				ResolveCanonicalBootstrapServersOnly: true,
@@ -71,6 +72,44 @@ func TestLoadConfig(t *testing.T) {
 			id: component.NewIDWithName(metadata.Type, "logs"),
 			expected: &Config{
 				Topic:             "logs",
+				TopicRegex:        "",
+				Encoding:          "direct",
+				Brokers:           []string{"coffee:123", "foobar:456"},
+				ClientID:          "otel-collector",
+				GroupID:           "otel-collector",
+				InitialOffset:     "earliest",
+				SessionTimeout:    45 * time.Second,
+				HeartbeatInterval: 15 * time.Second,
+				Authentication: kafka.Authentication{
+					TLS: &configtls.ClientConfig{
+						Config: configtls.Config{
+							CAFile:   "ca.pem",
+							CertFile: "cert.pem",
+							KeyFile:  "key.pem",
+						},
+					},
+				},
+				Metadata: kafkaexporter.Metadata{
+					Full: true,
+					Retry: kafkaexporter.MetadataRetry{
+						Max:     10,
+						Backoff: time.Second * 5,
+					},
+				},
+				AutoCommit: AutoCommit{
+					Enable:   true,
+					Interval: 1 * time.Second,
+				},
+				MinFetchSize:     1,
+				DefaultFetchSize: 1048576,
+				MaxFetchSize:     0,
+			},
+		},
+		{
+			id: component.NewIDWithName(metadata.Type, "topics_err"),
+			expected: &Config{
+				Topic:             "logs",
+				TopicRegex:        "logs[0-9]",
 				Encoding:          "direct",
 				Brokers:           []string{"coffee:123", "foobar:456"},
 				ClientID:          "otel-collector",
@@ -114,7 +153,11 @@ func TestLoadConfig(t *testing.T) {
 			require.NoError(t, err)
 			require.NoError(t, sub.Unmarshal(cfg))
 
-			assert.NoError(t, component.ValidateConfig(cfg))
+			if tt.id.String() == "kafka/topics_err" {
+				assert.EqualError(t, component.ValidateConfig(cfg), "only one setting 'topic' or 'topics_regex' can be used")
+			} else {
+				assert.NoError(t, component.ValidateConfig(cfg))
+			}
 			assert.Equal(t, tt.expected, cfg)
 		})
 	}

receiver/kafkareceiver/go.mod

@@ -28,6 +28,7 @@ require (
 	go.opentelemetry.io/collector/receiver v0.115.1-0.20241206185113-3f3e208e71b8
 	go.opentelemetry.io/collector/receiver/receivertest v0.115.1-0.20241206185113-3f3e208e71b8
 	go.opentelemetry.io/collector/semconv v0.115.1-0.20241206185113-3f3e208e71b8
+	go.opentelemetry.io/collector/service v0.111.0
 	go.opentelemetry.io/otel v1.32.0
 	go.opentelemetry.io/otel/metric v1.32.0
 	go.opentelemetry.io/otel/sdk/metric v1.32.0
@@ -88,6 +89,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/relvacode/iso8601 v1.6.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect