From 42a102af538ff08473b86a548da667d95ff26da6 Mon Sep 17 00:00:00 2001 From: SowmyaDixit Date: Fri, 19 Mar 2021 13:18:09 +0530 Subject: [PATCH 1/3] Issue #TG-717 feat: Secor on Kubernetes: Dockerfile changes --- Dockerfile | 1 + src/main/scripts/docker-entrypoint.sh | 35 +++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 042f128d2..d3ecb9877 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ ADD target/secor-*-bin.tar.gz $SECOR_HOME COPY src/main/scripts/docker-entrypoint.sh /docker-entrypoint.sh RUN find $SECOR_HOME -type d -exec chown -R secor:secor {} \; +RUN chown -R secor:secor /docker-entrypoint.sh RUN chmod +x /docker-entrypoint.sh USER secor diff --git a/src/main/scripts/docker-entrypoint.sh b/src/main/scripts/docker-entrypoint.sh index f7c677fee..55b360cf8 100644 --- a/src/main/scripts/docker-entrypoint.sh +++ b/src/main/scripts/docker-entrypoint.sh @@ -1,8 +1,8 @@ #!/bin/sh set -e -​ + # SECOR_CONFIG='' -​ + # if [ -z "$ZOOKEEPER_QUORUM" ]; then # echo "ZOOKEEPER_QUORUM variable not set, launch with -e ZOOKEEPER_QUORUM=zookeeper:2181" # exit 1 @@ -10,7 +10,7 @@ set -e # SECOR_CONFIG="$SECOR_CONFIG -Dzookeeper.quorum=$ZOOKEEPER_QUORUM" # echo "zookeeper.quorum=$ZOOKEEPER_QUORUM" # fi -​ + # if [ -z "$ZOOKEEPER_PATH" ]; then # echo "ZOOKEEPER_PATH variable not set, launch with -e ZOOKEEPER_PATH=/" # exit 1 @@ -18,7 +18,7 @@ set -e # SECOR_CONFIG="$SECOR_CONFIG -Dkafka.zookeeper.path=$ZOOKEEPER_PATH" # echo "kafka.zookeeper.path=$ZOOKEEPER_PATH" # fi -​ + # if [ ! -z "$KAFKA_SEED_BROKER_HOST" ]; then # SECOR_CONFIG="$SECOR_CONFIG -Dkafka.seed.broker.host=$KAFKA_SEED_BROKER_HOST" # echo "kafka.seed.broker.host=$KAFKA_SEED_BROKER_HOST" @@ -27,13 +27,13 @@ set -e # SECOR_CONFIG="$SECOR_CONFIG -Dkafka.seed.broker.port=$KAFKA_SEED_BROKER_PORT" # echo "kafka.seed.broker.port=$KAFKA_SEED_BROKER_PORT" # fi -​ + if [ ! -z "$SECOR_GROUP" ]; then SECOR_CONFIG="$SECOR_CONFIG -Dsecor.kafka.group=$SECOR_GROUP" echo "secor.kafka.group=$SECOR_GROUP" fi -​ -​ + + # if [ ! -z "$AWS_REGION" ]; then # SECOR_CONFIG="$SECOR_CONFIG -Daws.region=$AWS_REGION" # echo "aws.region=$AWS_REGION" @@ -67,7 +67,7 @@ fi # SECOR_CONFIG="$SECOR_CONFIG -Dschema.registry.url=$SECOR_SCHEMA_REGISTRY" # echo "schema.registry.url=$SECOR_SCHEMA_REGISTRY" # fi -​ + # if [ ! -z "$MESSAGE_TIMESTAMP_NAME" ]; then # SECOR_CONFIG="$SECOR_CONFIG -Dmessage.timestamp.name=$MESSAGE_TIMESTAMP_NAME" # echo "message.timestamp.name=$MESSAGE_TIMESTAMP_NAME" @@ -80,7 +80,7 @@ fi # SECOR_CONFIG="$SECOR_CONFIG -Dsecor.parser.timezone=$SECOR_PARSER_TIMEZONE" # echo "secor.parser.timezone=$SECOR_PARSER_TIMEZONE" # fi -​ + # if [ ! -z "$CLOUD_SERVICE" ]; then # SECOR_CONFIG="$SECOR_CONFIG -Dcloud.service=$CLOUD_SERVICE" # echo "cloud.service=$CLOUD_SERVICE" @@ -97,7 +97,7 @@ fi # SECOR_CONFIG="$SECOR_CONFIG -Dsecor.gs.path=$SECOR_GS_PATH" # echo "secor.gs.path=$SECOR_GS_PATH" # fi -​ + # if [ ! -z "$SECOR_MAX_FILE_BYTES" ]; then # SECOR_CONFIG="$SECOR_CONFIG -Dsecor.max.file.size.bytes=$SECOR_MAX_FILE_BYTES" # echo "secor.max.file.size.bytes=$SECOR_MAX_FILE_BYTES" @@ -106,8 +106,8 @@ fi # SECOR_CONFIG="$SECOR_CONFIG -Dsecor.max.file.age.seconds=$SECOR_MAX_FILE_SECONDS" # echo "secor.max.file.age.seconds=$SECOR_MAX_FILE_SECONDS" # fi -​ -​ + + # if [ ! -z "$SECOR_KAFKA_TOPIC_FILTER" ]; then # SECOR_CONFIG="$SECOR_CONFIG -Dsecor.kafka.topic_filter=$SECOR_KAFKA_TOPIC_FILTER" # echo "secor.kafka.topic_filter=$SECOR_KAFKA_TOPIC_FILTER" @@ -120,15 +120,14 @@ fi # SECOR_CONFIG="$SECOR_CONFIG -Dsecor.message.parser.class=$SECOR_MESSAGE_PARSER" # echo "secor.message.parser.class=$SECOR_MESSAGE_PARSER" # fi -# SECOR_CONFIG="$SECOR_CONFIG $SECOR_EXTRA_OPTS" -​ -​ +# SECOR_CONFIG="$SECOR_CONFIG $SECOR_EXTRA_OPTS"​ + cd /opt/secor -​ -​ + + DEFAULT_CLASSPATH="*:lib/*" CLASSPATH=${CLASSPATH:-$DEFAULT_CLASSPATH} -​ + # exec java -Xmx${JVM_MEMORY:-512m} $JAVA_OPTS -ea -Dsecor_group=${SECOR_GROUP:-partition} -Dlog4j.configuration=file:${LOG4J_CONFIGURATION:-log4j.docker.properties} \ # -Dconfig=${CONFIG_FILE:-secor.prod.partition.properties} $SECOR_CONFIG \ # -cp $CLASSPATH ${SECOR_MAIN_CLASS:-com.pinterest.secor.main.ConsumerMain} From db5f372e91e467d358b55f81d8465ac6ef080768 Mon Sep 17 00:00:00 2001 From: SowmyaDixit Date: Fri, 19 Mar 2021 16:54:24 +0530 Subject: [PATCH 2/3] Issue #TG-717 feat: Secor on Kubernetes: Dockerfile changes --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d3ecb9877..7a0473478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ADD target/secor-*-bin.tar.gz $SECOR_HOME COPY src/main/scripts/docker-entrypoint.sh /docker-entrypoint.sh RUN find $SECOR_HOME -type d -exec chown -R secor:secor {} \; -RUN chown -R secor:secor /docker-entrypoint.sh +RUN chown secor:secor /docker-entrypoint.sh RUN chmod +x /docker-entrypoint.sh USER secor From 149e25bce78ac6c68f1f493482b7055c745388da Mon Sep 17 00:00:00 2001 From: SowmyaDixit Date: Tue, 23 Mar 2021 10:28:41 +0530 Subject: [PATCH 3/3] Issue #TG-717 feat: Secor on Kubernetes: Set auto commit offset to false --- src/main/java/com/pinterest/secor/common/SecorKafkaClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/pinterest/secor/common/SecorKafkaClient.java b/src/main/java/com/pinterest/secor/common/SecorKafkaClient.java index 363a61660..0366aa976 100644 --- a/src/main/java/com/pinterest/secor/common/SecorKafkaClient.java +++ b/src/main/java/com/pinterest/secor/common/SecorKafkaClient.java @@ -109,7 +109,7 @@ public void init(SecorConfig config) { mPollTimeout = config.getNewConsumerPollTimeoutSeconds(); Properties props = new Properties(); props.put("bootstrap.servers", config.getKafkaSeedBrokerHost() + ":" + config.getKafkaSeedBrokerPort()); - props.put("enable.auto.commit", true); + props.put("enable.auto.commit", false); props.put("auto.offset.reset", "earliest"); props.put("key.deserializer", ByteArrayDeserializer.class); props.put("value.deserializer", ByteArrayDeserializer.class);