From 4ae89bbb15a0569118f67093ec0de272da556ab0 Mon Sep 17 00:00:00 2001 From: DawidCh Date: Mon, 3 Jul 2017 13:14:54 +0200 Subject: [PATCH 1/7] Modified version of dockerfile to wurstmeister/kafka:latest --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b72b95c..b89865c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM wurstmeister/kafka:0.10.0.1 +FROM wurstmeister/kafka:0.10.2.1 MAINTAINER CloudTrackInc From afe34f77579c787b44a40f5b6d91ca8b84c51d35 Mon Sep 17 00:00:00 2001 From: DawidCh Date: Mon, 3 Jul 2017 13:23:36 +0200 Subject: [PATCH 2/7] Modified version of kafka to 0.10.2.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 478cfb0..28e03d2 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ repositories { dependencies { compile 'org.scala-lang:scala-library:2.11.7' compile 'com.101tec:zkclient:0.7' - compile 'org.apache.kafka:kafka_2.11:0.10.0.1' + compile 'org.apache.kafka:kafka_2.11:0.10.2.1' compile 'org.scala-lang:scala-parser-combinators:2.11.0-M4' testCompile group: 'junit', name: 'junit', version: '4.11' } From d459419ec77e89292507757249d26a34c5705939 Mon Sep 17 00:00:00 2001 From: DawidCh Date: Tue, 4 Jul 2017 11:02:37 +0200 Subject: [PATCH 3/7] Modified image sourcing and maintainer --- Dockerfile | 2 +- kafka-rc.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b89865c..b0e63ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM wurstmeister/kafka:0.10.2.1 -MAINTAINER CloudTrackInc +MAINTAINER dawidCh RUN java -version diff --git a/kafka-rc.yaml b/kafka-rc.yaml index ec18133..85f8ce5 100644 --- a/kafka-rc.yaml +++ b/kafka-rc.yaml @@ -15,7 +15,7 @@ spec: spec: containers: - name: kafka - image: cloudtrackinc/kubernetes-kafka:latest + image: dawidch/kubernetes-kafka:0.10.2.1 ports: - containerPort: 9092 env: @@ -36,4 +36,4 @@ spec: - name: KAFKA_ZOOKEEPER_CONNECT value: kafka-zoo-svc:2181 - name: KAFKA_CREATE_TOPICS - value: demo-topic:16:1 \ No newline at end of file + value: demo-topic:16:1 From 843e61f7112b65edfd7db97ed959e01ef5178ccc Mon Sep 17 00:00:00 2001 From: Dawid Chojnacki Date: Tue, 4 Jul 2017 11:52:46 +0200 Subject: [PATCH 4/7] Modified scala version to 2.12.2 --- build.gradle | 12 ++++++------ gradle/wrapper/gradle-wrapper.properties | 4 ++-- src/main/scala/kafka/admin/AutoExpand.scala | 20 ++++++++------------ 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 28e03d2..9f68591 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ group 'org.cloudtrack.kubetrack' -version '1.0-SNAPSHOT' +version '2.0-SNAPSHOT' apply plugin: 'java' @@ -8,11 +8,11 @@ repositories { } dependencies { - compile 'org.scala-lang:scala-library:2.11.7' - compile 'com.101tec:zkclient:0.7' - compile 'org.apache.kafka:kafka_2.11:0.10.2.1' - compile 'org.scala-lang:scala-parser-combinators:2.11.0-M4' - testCompile group: 'junit', name: 'junit', version: '4.11' + compile 'org.scala-lang:scala-library:2.12.2' + compile 'com.101tec:zkclient:0.10' + compile 'org.apache.kafka:kafka_2.12:0.10.2.1' + compile 'org.scala-lang.modules:scala-parser-combinators_2.12:1.0.4' + testCompile group: 'junit', name: 'junit', version: '4.12' } allprojects { diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 993b571..46bbe90 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Fri Mar 11 14:44:25 MSK 2016 +#Tue Jul 04 11:45:52 CEST 2017 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-2.11-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-2.11-all.zip diff --git a/src/main/scala/kafka/admin/AutoExpand.scala b/src/main/scala/kafka/admin/AutoExpand.scala index da9330e..dcf7e5b 100644 --- a/src/main/scala/kafka/admin/AutoExpand.scala +++ b/src/main/scala/kafka/admin/AutoExpand.scala @@ -1,20 +1,17 @@ package kafka.admin import java.util.concurrent.{CountDownLatch, LinkedBlockingDeque, TimeUnit} +import java.util.{Collections, List => JList} import joptsimple.OptionParser -import kafka.cluster.BrokerEndPoint -import kafka.common.{TopicAndPartition, AdminCommandFailedException} +import kafka.common.{AdminCommandFailedException, TopicAndPartition} import kafka.utils._ -import org.apache.kafka.common.protocol.SecurityProtocol +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener} import org.apache.kafka.common.security.JaasUtils +import org.apache.zookeeper.CreateMode -import collection._ import _root_.scala.collection.JavaConversions._ -import java.util.{Collections, List => JList} - -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener} -import org.apache.zookeeper.CreateMode +import scala.collection._ object AutoExpandCommand { @@ -48,7 +45,6 @@ object AutoExpandCommand { val mNode = zkUtils.zkClient.createEphemeralSequential(KAFKA_POD_MASTER+"/pod-", uid) val index = sequnce(mNode) case class Broker(index: Int, id: String) extends Ordered[Broker] { - import scala.math.Ordered.orderingToOrdered def compare(that: Broker): Int = this.index.compareTo(that.index) } @@ -184,8 +180,8 @@ object AutoExpandCommand { } var (partitionsToBeReassigned, currentAssignments) = generateAssignment(zkUtils, newBrokersIds, topics, true) - println("Current partition replica assignment\n\n%s".format(zkUtils.formatAsReassignmentJson(currentAssignments))) - println("Proposed partition reassignment configuration\n\n%s".format(zkUtils.formatAsReassignmentJson(partitionsToBeReassigned))) + println("Current partition replica assignment\n\n%s".format(ZkUtils.formatAsReassignmentJson(currentAssignments))) + println("Proposed partition reassignment configuration\n\n%s".format(ZkUtils.formatAsReassignmentJson(partitionsToBeReassigned))) val exec = new ReassignPartitionsCommand(zkUtils, partitionsToBeReassigned) if (exec.reassignPartitions()){ var inProgress = true @@ -215,7 +211,7 @@ object AutoExpandCommand { attemt+=1 } if (inProgress){ - println("Timeout reassignment partitions:\n\n%s".format(zkUtils.formatAsReassignmentJson(currentAssignments))) + println("Timeout reassignment partitions:\n\n%s".format(ZkUtils.formatAsReassignmentJson(currentAssignments))) } else{ println("Reassignment compleated!!!") } From b718c5ef77bedc71a8678e662b09e3096abba8fd Mon Sep 17 00:00:00 2001 From: Dawid Chojnacki Date: Tue, 4 Jul 2017 11:55:04 +0200 Subject: [PATCH 5/7] Modified version of build. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b0e63ae..61a2f85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN java -version ADD . /tmp/build RUN cd /tmp/build && \ ./gradlew -Dorg.gradle.native=false build && \ - cp build/libs/kubernetes-expander-1.0-SNAPSHOT.jar $KAFKA_HOME/libs/ + cp build/libs/kubernetes-expander-2.0-SNAPSHOT.jar $KAFKA_HOME/libs/ ADD kafka-autoextend-partitions.sh /usr/bin/kafka-autoextend-partitions.sh ADD kafka-kubernetes-start.sh /usr/bin/kafka-kubernetes-start.sh From d13f13d80d59b5235f8eebd4ff34876c815c74b0 Mon Sep 17 00:00:00 2001 From: Dawid Chojnacki Date: Tue, 4 Jul 2017 13:42:55 +0200 Subject: [PATCH 6/7] Enabling topic deletion delete.topic.enable = true in config/server.properties. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 61a2f85..10023ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,6 @@ RUN cd /tmp/build && \ ADD kafka-autoextend-partitions.sh /usr/bin/kafka-autoextend-partitions.sh ADD kafka-kubernetes-start.sh /usr/bin/kafka-kubernetes-start.sh +RUN echo delete.topic.enable=true >> /opt/kafka/config/server.properties CMD ["kafka-kubernetes-start.sh"] From 931dd59b389564ab179d4b392b6c81aebae8c8f5 Mon Sep 17 00:00:00 2001 From: DawidCh Date: Thu, 6 Jul 2017 11:23:38 +0200 Subject: [PATCH 7/7] Added two config entries --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 10023ca..2587f9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,6 @@ RUN cd /tmp/build && \ ADD kafka-autoextend-partitions.sh /usr/bin/kafka-autoextend-partitions.sh ADD kafka-kubernetes-start.sh /usr/bin/kafka-kubernetes-start.sh RUN echo delete.topic.enable=true >> /opt/kafka/config/server.properties +RUN listeners=PLAINTEXTX://0.0.0.0:9092 >> /opt/kafka/config/server.properties CMD ["kafka-kubernetes-start.sh"]