bump to use 0.10 Kafka Client (logstash-plugins#72)

trsmith2 · Jun 23, 2016 · 49f80fe · 49f80fe
1 parent 2fd6cb9
commit 49f80fe
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 29 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,28 +2,13 @@
 sudo: false
 language: ruby
 cache: bundler
+jdk: oraclejdk8
 rvm:
-- jruby-1.7.25
-before_install:
-- curl -s -o kafka.tgz http://ftp.wayne.edu/apache/kafka/0.9.0.1/kafka_2.11-0.9.0.1.tgz
-- mkdir kafka && tar xzf kafka.tgz -C kafka --strip-components 1
-- kafka/bin/zookeeper-server-start.sh kafka/config/zookeeper.properties &
-- kafka/bin/kafka-server-start.sh kafka/config/server.properties &
-- sleep 3
-- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic
-  topic1 --zookeeper localhost:2181
-- kafka/bin/kafka-topics.sh --create --partitions 2 --replication-factor 1 --topic
-  topic2 --zookeeper localhost:2181
-- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic
-  gzip_topic --zookeeper localhost:2181
-- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic
-  snappy_topic --zookeeper localhost:2181
-- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic
-  lz4_topic --zookeeper localhost:2181
-- kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic
-  topic3 --zookeeper localhost:2181
+  - jruby-1.7.25
+env:
+  - KAFKA_VERSION=0.10.0.0
+before_install: ./kafka_test_setup.sh
 before_script:
-- bundle exec rake vendor
-- bundle exec rake install_jars
+  - bundle exec rake vendor
+  - bundle exec rake install_jars
 script: bundle exec rspec && bundle exec rspec --tag integration
-jdk: oraclejdk8
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,5 @@
+## 5.0.0
+  - Kafka 0.10 broker producer
 ## 4.0.0
   - Republish all the gems under jruby.
   - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141

diff --git a/kafka_test_setup.sh b/kafka_test_setup.sh
@@ -0,0 +1,11 @@
+curl -s -o kafka.tgz "http://ftp.wayne.edu/apache/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz"
+mkdir kafka && tar xzf kafka.tgz -C kafka --strip-components 1
+kafka/bin/zookeeper-server-start.sh kafka/config/zookeeper.properties &
+kafka/bin/kafka-server-start.sh kafka/config/server.properties &
+sleep 3
+kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic topic1 --zookeeper localhost:2181
+kafka/bin/kafka-topics.sh --create --partitions 2 --replication-factor 1 --topic topic2 --zookeeper localhost:2181
+kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic gzip_topic --zookeeper localhost:2181
+kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic snappy_topic --zookeeper localhost:2181
+kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic lz4_topic --zookeeper localhost:2181
+kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic topic3 --zookeeper localhost:2181
diff --git a/lib/logstash/outputs/kafka.rb b/lib/logstash/outputs/kafka.rb
@@ -93,8 +93,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :retry_backoff_ms, :validate => :number, :default => 100
   # The size of the TCP send buffer to use when sending data.
   config :send_buffer_bytes, :validate => :number, :default => 131072
-  # Enable SSL/TLS secured communication to Kafka broker. Note that secure communication 
-  # is only available with a broker running v0.9 of Kafka.
+  # Enable SSL/TLS secured communication to Kafka broker.
   config :ssl, :validate => :boolean, :default => false
   # The JKS truststore path to validate the Kafka broker's certificate.
   config :ssl_truststore_location, :validate => :path
@@ -117,8 +116,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   public
   def register
-    LogStash::Logger.setup_log4j(@logger)
-
     @producer = create_producer
     @codec.on_event do |event, data|
       begin
@@ -170,7 +167,7 @@ def create_producer
       props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s)
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s)
       props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, value_serializer)
-      
+
       if ssl
         if ssl_truststore_location.nil?
           raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled"

diff --git a/logstash-output-kafka.gemspec b/logstash-output-kafka.gemspec
@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '4.0.0'
+  s.version         = '5.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
 
-  s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.9.0.1'"
+  s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.10.0.0'"
   s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'"
 
   s.add_development_dependency 'jar-dependencies', '~> 0.3.2'