diff --git a/.travis.yml b/.travis.yml index 6af216f..b2c68b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,28 +2,13 @@ sudo: false language: ruby cache: bundler +jdk: oraclejdk8 rvm: -- jruby-1.7.25 -before_install: -- curl -s -o kafka.tgz http://ftp.wayne.edu/apache/kafka/0.9.0.1/kafka_2.11-0.9.0.1.tgz -- mkdir kafka && tar xzf kafka.tgz -C kafka --strip-components 1 -- kafka/bin/zookeeper-server-start.sh kafka/config/zookeeper.properties & -- kafka/bin/kafka-server-start.sh kafka/config/server.properties & -- sleep 3 -- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic - topic1 --zookeeper localhost:2181 -- kafka/bin/kafka-topics.sh --create --partitions 2 --replication-factor 1 --topic - topic2 --zookeeper localhost:2181 -- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic - gzip_topic --zookeeper localhost:2181 -- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic - snappy_topic --zookeeper localhost:2181 -- kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic - lz4_topic --zookeeper localhost:2181 -- kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic - topic3 --zookeeper localhost:2181 + - jruby-1.7.25 +env: + - KAFKA_VERSION=0.10.0.0 +before_install: ./kafka_test_setup.sh before_script: -- bundle exec rake vendor -- bundle exec rake install_jars + - bundle exec rake vendor + - bundle exec rake install_jars script: bundle exec rspec && bundle exec rspec --tag integration -jdk: oraclejdk8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6850a86..b1f35ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 5.0.0 + - Kafka 0.10 broker producer ## 4.0.0 - Republish all the gems under jruby. - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 diff --git a/kafka_test_setup.sh b/kafka_test_setup.sh new file mode 100755 index 0000000..33c16ae --- /dev/null +++ b/kafka_test_setup.sh @@ -0,0 +1,11 @@ +curl -s -o kafka.tgz "http://ftp.wayne.edu/apache/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz" +mkdir kafka && tar xzf kafka.tgz -C kafka --strip-components 1 +kafka/bin/zookeeper-server-start.sh kafka/config/zookeeper.properties & +kafka/bin/kafka-server-start.sh kafka/config/server.properties & +sleep 3 +kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic topic1 --zookeeper localhost:2181 +kafka/bin/kafka-topics.sh --create --partitions 2 --replication-factor 1 --topic topic2 --zookeeper localhost:2181 +kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic gzip_topic --zookeeper localhost:2181 +kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic snappy_topic --zookeeper localhost:2181 +kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --topic lz4_topic --zookeeper localhost:2181 +kafka/bin/kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic topic3 --zookeeper localhost:2181 diff --git a/lib/logstash/outputs/kafka.rb b/lib/logstash/outputs/kafka.rb index 5555808..237f166 100644 --- a/lib/logstash/outputs/kafka.rb +++ b/lib/logstash/outputs/kafka.rb @@ -93,8 +93,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base config :retry_backoff_ms, :validate => :number, :default => 100 # The size of the TCP send buffer to use when sending data. config :send_buffer_bytes, :validate => :number, :default => 131072 - # Enable SSL/TLS secured communication to Kafka broker. Note that secure communication - # is only available with a broker running v0.9 of Kafka. + # Enable SSL/TLS secured communication to Kafka broker. config :ssl, :validate => :boolean, :default => false # The JKS truststore path to validate the Kafka broker's certificate. config :ssl_truststore_location, :validate => :path @@ -117,8 +116,6 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base public def register - LogStash::Logger.setup_log4j(@logger) - @producer = create_producer @codec.on_event do |event, data| begin @@ -170,7 +167,7 @@ def create_producer props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s) props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, value_serializer) - + if ssl if ssl_truststore_location.nil? raise LogStash::ConfigurationError, "ssl_truststore_location must be set when SSL is enabled" diff --git a/logstash-output-kafka.gemspec b/logstash-output-kafka.gemspec index 24364ad..23dd997 100644 --- a/logstash-output-kafka.gemspec +++ b/logstash-output-kafka.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-kafka' - s.version = '4.0.0' + s.version = '5.0.0' s.licenses = ['Apache License (2.0)'] s.summary = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker' s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -19,7 +19,7 @@ Gem::Specification.new do |s| # Special flag to let us know this is actually a logstash plugin s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'} - s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.9.0.1'" + s.requirements << "jar 'org.apache.kafka:kafka-clients', '0.10.0.0'" s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.13'" s.add_development_dependency 'jar-dependencies', '~> 0.3.2'