-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconnect-mirror-maker-template.properties
100 lines (84 loc) · 4.7 KB
/
connect-mirror-maker-template.properties
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# See https://cwiki.apache.org/confluence/display/KAFKA/KIP-382%3A+MirrorMaker+2.0#KIP382:MirrorMaker2.0-ConnectorConfigurationProperties for all properties
# Also https://cwiki.apache.org/confluence/display/KAFKA/KIP-545:+support+automated+consumer+offset+sync+across+clusters+in+MM+2.0
# Check the source here: https://github.com/apache/kafka/blob/trunk/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorMakerConfig.java
# Kafka datacenters
clusters={{.Env.SOURCE_NAME}}, {{.Env.TARGET_NAME}}
{{.Env.SOURCE_NAME}}.bootstrap.servers={{.Env.SOURCE_BOOTSTRAP_SERVERS}}
{{.Env.TARGET_NAME}}.bootstrap.servers={{.Env.TARGET_BOOTSTRAP_SERVERS}}
# This configuration is for active -> passive mirroring
{{.Env.SOURCE_NAME}}->{{.Env.TARGET_NAME}}.enabled=true
{{.Env.TARGET_NAME}}->{{.Env.SOURCE_NAME}}.enabled=false
topics={{.Env.TOPICS}}
groups={{.Env.GROUPS}}
# Keep topic names (do not append cluster prefix)
{{- if eq .Env.TARGET_TOPICS_ADD_CLUSTER_PREFIX "true" }}
replication.policy.class=org.apache.kafka.connect.mirror.DefaultReplicationPolicy
{{- else}}
replication.policy.class=org.apache.kafka.connect.mirror.IdentityReplicationPolicy
{{- end}}
topics.exclude=.*[\-\.]internal, .*\.replica, __consumer_offsets
groups.exclude=console-consumer-.*, connect-.*, __.*
#target and source configurations. Default value for the following settings is 3.
#If you want more details about those internal configurations, please see https://docs.confluent.io/home/connect/userguide.html#kconnect-internal-topics
#and https://docs.confluent.io/platform/current/connect/references/allconfigs.html#distributed-worker-configuration
{{.Env.TARGET_NAME}}.config.storage.replication.factor=3
{{.Env.SOURCE_NAME}}.config.storage.replication.factor=3
#
{{.Env.TARGET_NAME}}.offset.storage.replication.factor=3
{{.Env.SOURCE_NAME}}.offset.storage.replication.factor=3
#
{{.Env.TARGET_NAME}}.status.storage.replication.factor=3
{{.Env.SOURCE_NAME}}.status.storage.replication.factor=3
# MirrorMaker configuration. Default value for the following settings is 3
offset-syncs.topic.replication.factor=3
heartbeats.topic.replication.factor=3
checkpoints.topic.replication.factor=3
tasks.max=10
# Make sure offsets are synchronized fully. This makes sure the consumer groups get eventually in sync.
offset.lag.max=0
replication.factor=3
refresh.topics.enabled=true
refresh.topics.interval.seconds=30
sync.group.offsets.interval.seconds=10
emit.checkpoints.interval.seconds=10
sync.topic.configs.enabled={{.Env.SYNC_TOPIC_CONFIGS}}
sync.topic.acls.enabled={{.Env.SYNC_TOPIC_ACLS}}
# Enable heartbeats and checkpoints
{{.Env.SOURCE_NAME}}->{{.Env.TARGET_NAME}}.emit.checkpoints.enabled=true
{{.Env.SOURCE_NAME}}->{{.Env.TARGET_NAME}}.sync.group.offsets.enabled = true
{{.Env.SOURCE_NAME}}->{{.Env.TARGET_NAME}}.emit.heartbeats.enabled=true
{{.Env.SOURCE_NAME}}->{{.Env.TARGET_NAME}}.sync.group.offsets.interval.seconds=10
{{.Env.TARGET_NAME}}->{{.Env.SOURCE_NAME}}.emit.heartbeats.enabled=false
{{.Env.TARGET_NAME}}->{{.Env.SOURCE_NAME}}.emit.checkpoints.enabled=false
{{.Env.TARGET_NAME}}->{{.Env.SOURCE_NAME}}.sync.group.offsets.enabled = false
# allow a generous 1GB per message, as the default is 1MB
{{.Env.SOURCE_NAME}}.consumer.max.request.size=1073741824
{{.Env.SOURCE_NAME}}.producer.max.request.size=1073741824
{{.Env.TARGET_NAME}}.consumer.max.request.size=1073741824
{{.Env.TARGET_NAME}}.producer.max.request.size=1073741824
# SSL properties for connecting to SSL enabled source clusters
{{ if eq .Env.SOURCE_SSL_ENABLE "true" }}
{{.Env.SOURCE_NAME}}.ssl.enabled.protocols=TLSv1.2
{{.Env.SOURCE_NAME}}.security.inter.broker.protocol=SSL
{{.Env.SOURCE_NAME}}.security.protocol=SSL
{{.Env.SOURCE_NAME}}.ssl.endpoint.identification.algorithm=
{{ if eq .Env.SOURCE_TRUSTSTORE_REQUIRED "true" }}
{{.Env.SOURCE_NAME}}.ssl.truststore.location={{.Env.SOURCE_TRUSTSTORE}}
{{.Env.SOURCE_NAME}}.ssl.truststore.password={{.Env.SOURCE_TRUSTSTORE_PASSWORD}}
{{ end }}
{{.Env.SOURCE_NAME}}.ssl.keystore.location={{.Env.SOURCE_KEYSTORE}}
{{.Env.SOURCE_NAME}}.ssl.keystore.password={{.Env.SOURCE_KEYSTORE_PASSWORD}}
{{ end }}
{{ if eq .Env.TARGET_SSL_ENABLE "true" }}
# SSL properties for connecting to SSL enabled target clusters
{{.Env.TARGET_NAME}}.security.inter.broker.protocol=SSL
{{.Env.TARGET_NAME}}.ssl.enabled.protocols=TLSv1.2
{{.Env.TARGET_NAME}}.security.protocol=SSL
{{.Env.TARGET_NAME}}.ssl.endpoint.identification.algorithm=
{{ if eq .Env.TARGET_TRUSTSTORE_REQUIRED "true" }}
{{.Env.TARGET_NAME}}.ssl.truststore.location={{.Env.TARGET_TRUSTSTORE}}
{{.Env.TARGET_NAME}}.ssl.truststore.password={{.Env.TARGET_TRUSTSTORE_PASSWORD}}
{{ end }}
{{.Env.TARGET_NAME}}.ssl.keystore.location={{.Env.TARGET_KEYSTORE}}
{{.Env.TARGET_NAME}}.ssl.keystore.password={{.Env.TARGET_KEYSTORE_PASSWORD}}
{{ end }}