Merge branch 'master' into path-params

Conflicts: manifests/config.pp spec/classes/packetbeat_spec.rb
corey-hammerton · Dec 28, 2017 · 56b724a · 56b724a
2 parents e2d5c81 + 5ba9df8
commit 56b724a
Show file tree

Hide file tree

Showing 6 changed files with 190 additions and 35 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,14 @@
 Changelog
 =========
 
+## [0.2.0]
+
+- Adding support for Packetbeat 6.0
+-- Removing unsupported sniffer type `pf_ring` from available `sniff_type` options
+-- Adding new parameter `major_version` to allow installation of 6.x packages from vendor
+-- Adding new optional parameter `queue` to configure internal queue settings
+- Parameter `queue_size` is only applicable if `major_version` is '5'
+
 ## [0.1.1](https://github.com/corey-hammerton/puppet-packetbeat/tree/0.1.1)
 
 - Fixing configuration file validation

diff --git a/README.md b/README.md
@@ -87,6 +87,21 @@ class{'packetbeat':
 
 [Network device configuation](https://www.elastic.co/guide/en/beats/packetbeat/current/configuration-interfaces.html) and [logging](https://www.elastic.co/guide/en/beats/packetbeat/current/configuration-logging.html) can be configured the same way. Please review the documentation of the [elastic website](https://www.elastic.co/guide/en/beats/packetbeat/current/index.html)
 
+### Upgrading to 6.0
+
+Version 0.2.0 of this module supports Packetbeat 6.0. Please review the [Packetbeat Changelog](https://www.elastic.co/guide/en/beats/libbeat/6.0/release-notes-6.0.0.html)
+for a full list of software changes and the Module Changelog for a list of module updates.
+
+To upgrade existing installations:
+
+```puppet
+class{'packetbeat':
+  major_version  => '6',
+  package_ensure => 'latest',
+  ...
+}
+```
+
 ### Processors
 
 Libbeat 5.0 and later include a feature for filtering/enhancing exported data
@@ -177,14 +192,18 @@ Installs and configures packetbeat.
 - `logging`: [Hash] Defines packetbeat's logging configuration, if not explicitly
   configured all logging output is forwarded to syslog on Linux nodes and file
   output on Windows. See the [docs](https://www.elastic.co/guide/en/beats/packetbeat/current/configuration-logging.html) for all available options.
+- `major_version`: [Enum] The major version of Packetbeat to install. Valid values
+  are '5' and '6'. (default: '5')
 - `manage_repo`: [Boolean] When false does not install the upstream repository
   to the node's package manager. (default: true)
 - `package_ensure`: [String] The desired state of the Package resources. Only
   applicable if `ensure` is 'present'. (default: 'present')
 - `processors`: [Array[Hash]] Add processors to the configuration to run on data
   before sending to the output. (default: undef)
+- `queue`: [Hash] Configure the internal queue in packetbeat before being consumed
+  by the output(s).
 - `queue_size`: [Integer] The queue size for single events in the processing
-  pipeline. (default: 1000)
+   pipeline. This is only applicable if `major_version` is '5'. (default: 1000)
 - `service_ensure`: [String] Determine the state of the packet beat service. Must
   be one of 'enabled', 'disabled', 'running', 'unmanaged'. (default: enabled)
 - `service_has_restart`: [Boolean] When true the Service resource issues the
@@ -193,9 +212,7 @@ Installs and configures packetbeat.
   environments can accept the default, on a physical interface the optimal value
   is the MTU size. (default: 65535)
 - `sniff_type`: [String] Configure the sniffer type, packet beat only supports
-  'pcap', 'af_packet' (Linux only, faster than 'pcap') and 'pf_ring' (Requires
-  a kernel module and a re-compilation of Packetbeat, not supported by Elastic).
-  (default: 'pcap')
+  'pcap', and 'af_packet' (Linux only, faster than 'pcap') (default: 'pcap')
 - `tags`: [Array] Optional list of tags to help group different logical properties
   easily. (default: undef)
 - `with_vlans`: [Boolean] If traffic contains VLAN tags all traffic is offset by

diff --git a/manifests/config.pp b/manifests/config.pp
@@ -9,29 +9,56 @@
     true    => undef,
     default => '/usr/share/packetbeat/bin/packetbeat -N -configtest -c %',
   }
-  $packetbeat_config = delete_undef_values({
-    'name'              => $packetbeat::beat_name,
-    'fields'            => $packetbeat::fields,
-    'fields_under_root' => $packetbeat::fields_under_root,
-    'logging'           => $packetbeat::logging,
-    'queue_size'        => $packetbeat::queue_size,
-    'tags'              => $packetbeat::tags,
-    'processors'        => $packetbeat::processors,
-    'packetbeat'        => {
-      'interfaces' => {
-        'device'  => $packetbeat::device,
-        'snaplen' => $packetbeat::snaplen,
-        'type'    => $packetbeat::sniff_type,
+  if $packetbeat::major_version == '5' {
+    $packetbeat_config = delete_undef_values({
+      'name'              => $packetbeat::beat_name,
+      'fields'            => $packetbeat::fields,
+      'fields_under_root' => $packetbeat::fields_under_root,
+      'logging'           => $packetbeat::logging,
+      'tags'              => $packetbeat::tags,
+      'processors'        => $packetbeat::processors,
+      'packetbeat'        => {
+        'interfaces' => {
+          'device'  => $packetbeat::device,
+          'snaplen' => $packetbeat::snaplen,
+          'type'    => $packetbeat::sniff_type,
+        },
+        'flows'      => {
+          'enabled' => $packetbeat::flow_enable,
+          'period'  => $packetbeat::flow_period,
+          'timeout' => $packetbeat::flow_timeout,
+        },
+        'protocols'  => $packetbeat::protocols,
+        'queue_size' => $packetbeat::queue_size,
       },
-      'flows'      => {
-        'enabled' => $packetbeat::flow_enable,
-        'period'  => $packetbeat::flow_period,
-        'timeout' => $packetbeat::flow_timeout,
+      'output'     => $packetbeat::outputs,
+    })
+  }
+  else {
+    $packetbeat_config = delete_undef_values({
+      'name'              => $packetbeat::beat_name,
+      'fields'            => $packetbeat::fields,
+      'fields_under_root' => $packetbeat::fields_under_root,
+      'logging'           => $packetbeat::logging,
+      'tags'              => $packetbeat::tags,
+      'processors'        => $packetbeat::processors,
+      'packetbeat'        => {
+        'interfaces' => {
+          'device'  => $packetbeat::device,
+          'snaplen' => $packetbeat::snaplen,
+          'type'    => $packetbeat::sniff_type,
+        },
+        'flows'      => {
+          'enabled' => $packetbeat::flow_enable,
+          'period'  => $packetbeat::flow_period,
+          'timeout' => $packetbeat::flow_timeout,
+        },
+        'protocols'  => $packetbeat::protocols,
+        'queue'      => $packetbeat::queue,
       },
-      'protocols'  => $packetbeat::protocols,
-    },
-    'output'     => $packetbeat::outputs,
-  })
+      'output'     => $packetbeat::outputs,
+    })
+  }
 
   if $packetbeat::sniff_type == 'af_packet' {
     $af_packet_config = delete_undef_values({

diff --git a/manifests/init.pp b/manifests/init.pp
@@ -67,6 +67,10 @@
 # [Hash] The configuration section of `packetbeat.yml` for configuring the
 # logging output.
 #
+# * `major_version`
+# [Enum] The major version of Packetbeat to install from vendor repositories.
+# Valid values are '5' and '6'. (default: '5')
+#
 # * `manage_repo`
 # [Boolean] Weather the upstream (elastic) repo should be configured or
 # not. (default: true)
@@ -79,9 +83,15 @@
 # enhancing or additional decoding of data before being sent to the
 # output.
 #
+# * 'queue`
+# [Hash] Configure the internal queue before being consumed by the 
+# output(s) in bulk transactions. As of 6.0 only a memory queue is
+# available, all settings must be configured by example: { 'mem' => {...}}.
+#
 # * `queue_size`
 # [Number] The internal queue size for single events in the processing
-# pipeline. (default: 1000)
+# pipeline. This is only applicable if $major_version is '5'.
+# (default: 1000)
 #
 # * `service_ensure`
 # [String] The desired state of the packetbeat service. Must be one of
@@ -97,7 +107,7 @@
 #
 # * `sniff_type`
 # [String] The sniffer type to use. Packet only has support for pcap,
-# af_packet and pf_ring. (default: 'pcap')
+# and af_packet. (default: 'pcap')
 #
 # * `tags`
 # Optional[Array] A list of values to include in the `tags` field in each published
@@ -174,14 +184,24 @@
       'rotateeverybytes' => 10485760,
     },
   },
+  Enum['5', '6'] $major_version                                       = '5',
   Boolean $manage_repo                                                = true,
   String $package_ensure                                              = 'present',
   Optional[Array[Hash]] $processors                                   = undef,
+  Hash $queue                                                         = {
+    'mem' => {
+      'events' => 4096,
+      'flush'  => {
+        'min_events' => 0,
+        'timeout'    => '0s',
+      },
+    },
+  },
   Integer $queue_size                                                 = 1000,
   Enum['enabled', 'disabled', 'running', 'unmanaged'] $service_ensure = 'enabled',
   Boolean $service_has_restart                                        = true,
   Integer $snaplen                                                    = 65535,
-  Enum['pcap', 'af_packet', 'pf_ring'] $sniff_type                    = 'pcap',
+  Enum['pcap', 'af_packet'] $sniff_type                               = 'pcap',
   Optional[Array[String]] $tags                                       = undef,
   Optional[Boolean] $with_vlans                                       = undef,
 ) {

diff --git a/manifests/repo.pp b/manifests/repo.pp
@@ -9,9 +9,14 @@
     'Debian': {
       include ::apt
 
+      $download_url = $packetbeat::major_version ? {
+        '6' => 'https://artifacts.elastic.co/packages/6.x/apt',
+        '5' => 'https://artifacts.elastic.co/packages/5.x/apt',
+      }
+
       if !defined(Apt::Source['beats']) {
         apt::source{'beats':
-          location => 'https://artifacts.elastic.co/packages/5.x/apt',
+          location => $download_url,
           release  => 'stable',
           repos    => 'main',
           key      => {
@@ -22,25 +27,35 @@
       }
     }
     'Redhat': {
+      $download_url = $packetbeat::major_version ? {
+        '6' => 'https://artifacts.elastic.co/packages/6.x/yum',
+        '5' => 'https://artifacts.elastic.co/packages/5.x/yum',
+      }
+
       if !defined(Yumrepo['beats']) {
         yumrepo{'beats':
           descr    => 'Elastic repository for 5.x packages',
-          baseurl  => 'https://artifacts.elastic.co/packages/5.x/yum',
+          baseurl  => $download_url,
           gpgcheck => 1,
           gpgkey   => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
           enabled  => 1,
         }
       }
     }
     'SuSe': {
+      $download_url = $packetbeat::major_version ? {
+        '6' => 'https://artifacts.elastic.co/packages/6.x/yum',
+        '5' => 'https://artifacts.elastic.co/packages/5.x/yum',
+      }
+
       exec { 'topbeat_suse_import_gpg':
         command => '/usr/bin/rpmkeys --import https://artifacts.elastic.co/GPG-KEY-elasticsearch',
         unless  => '/usr/bin/test $(rpm -qa gpg-pubkey | grep -i "D88E42B4" | wc -l) -eq 1 ',
         notify  => [ Zypprepo['beats'] ],
       }
       if !defined (Zypprepo['beats']) {
         zypprepo{'beats':
-          baseurl     => 'https://artifacts.elastic.co/packages/5.x/yum',
+          baseurl     => $download_url,
           enabled     => 1,
           autorefresh => 1,
           name        => 'beats',

diff --git a/spec/classes/packetbeat_spec.rb b/spec/classes/packetbeat_spec.rb
@@ -121,6 +121,54 @@
             )
           end
         end
+
+        describe 'with major_version = 6' do
+          let(:params) { { 'major_version' => '6' } }
+
+          case os_facts[:osfamily]
+          when 'RedHat'
+            it do
+              is_expected.to contain_yumrepo('beats').with(
+                baseurl: 'https://artifacts.elastic.co/packages/6.x/yum',
+                enabled: 1,
+                gpgcheck: 1,
+                gpgkey: 'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
+              )
+            end
+          when 'Debian'
+            it { is_expected.to contain_class('apt') }
+
+            it do
+              is_expected.to contain_apt__source('beats').with(
+                location: 'https://artifacts.elastic.co/packages/6.x/apt',
+                release: 'stable',
+                repos: 'main',
+                key: {
+                  'id' => '46095ACC8548582C1A2699A9D27D666CD88E42B4',
+                  'source' => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
+                },
+              )
+            end
+          when 'SuSe'
+            it do
+              is_expected.to contain_zypprepo('beats').with(
+                baseurl: 'https://artifacts.elastic.co/packages/6.x/yum',
+                autorefresh: 1,
+                enabled: 1,
+                gpgcheck: 1,
+                gpgkey: 'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
+                name: 'beats',
+                type: 'yum',
+              )
+            end
+          end
+        end
+
+        describe 'with major_version = idontknow' do
+          let(:params) { { 'major_version' => 'idontknow' } }
+
+          it { is_expected.not_to compile }
+        end
       end
 
       describe 'packetbeat::service' do
@@ -242,25 +290,45 @@
       context 'with path_conf param' do
         let(:params) { { 'path_conf' => '/etc/packetbeat' } }
 
-        it { is_expected.to raise_error(Puppet::Error) }
+        it { is_expected.not_to compile }
       end
 
       context 'with path_data param' do
         let(:params) { { 'path_data' => '/var/lib/packetbeat' } }
 
-        it { is_expected.to raise_error(Puppet::Error) }
+        it { is_expected.not_to compile }
       end
 
       context 'with path_home param' do
         let(:params) { { 'path_home' => '/usr/share/packetbeat' } }
 
-        it { is_expected.to raise_error(Puppet::Error) }
+        it { is_expected.not_to compile }
       end
 
       context 'with path_logs param' do
         let(:params) { { 'path_logs' => '/var/lgs/packetbeat' } }
 
-        it { is_expected.to raise_error(Puppet::Error) }
+        it { is_expected.not_to compile }
+      end
+
+      context 'with sniff_type = pf_ring' do
+        let :params do
+          {
+            outputs:     {
+              'elasticsearch' => {
+                'hosts' => ['http://localhost:9200'],
+              },
+            },
+            protocols:   {
+              'icmp' => {
+                'enabled' => true,
+              },
+            },
+            sniff_type:  'pf_ring',
+          }
+        end
+
+        it { is_expected.not_to compile }
       end
     end
   end