From 427195ebcd4e469fd1104b0905b39136fbed72d1 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 16 Aug 2019 10:39:42 +0200 Subject: [PATCH] Squashed '.puppet/modules/filebeat/' changes from 351db281..96a95cc9 96a95cc9 Release 4.0.5 e120ab6b Setup module specific facts 84760c17 Restore deploy step in Travis 16dd9867 Update pdk version 1b58de0e When upgrading to Filebeat 7.x you should set the major_version to 7 otherwise the configuration tests will fail to execute. (#221) 2b0b6103 Update metadata versions (#220) 9925e5bf Release v4.0.4 d1b98521 Adding support for config file validation for version 7 on Windows (#219) 96aaec44 Release version 4.0.3 bfb79da1 Adding support for version check of Windows filebeat version 5 though 7 (#218) bc676580 Release v4.0.2 55023038 Clear the yum cache after making changes to the yum repo 0c0b077c Require the puppetlabs/yumrepo_core module 6756f005 Release 4.0.1 045f112c Fix regression on fresh installs (#216) 869f5d74 Update pdk and fix unit tests (#213) 49f45602 Release 4.0.0 c4a11770 Add unreleased tag to 4.0.0 a1b47e2b Remove queue_size 49430962 Add support for http metrics and filebeat 7 33507880 Convert prospector to input a60f6954 Prospectors depends on filebeat.yml (#211) 0a654dd8 Don't test against Puppet 4 9575d782 Document adding registry_flush d179388a Merge branch 'add_registry_flush' 3724d1b2 Make registry_flush optional ea3855fb Allow strings for clean_inactive 757ab26a Merge remote-tracking branch 'pcfens/master' into add_registry_flush 9138c8f9 Release v3.4.0 c902e924 Bring changelog up to date 796eb2a1 Fix filebeat::prospector::fields_under_root (#205) bca8323c Add a filebeat.config.modules section (#204) fdc28f97 Support filebeat.registry_flush in filebeat.yml git-subtree-dir: .puppet/modules/filebeat git-subtree-split: 96a95cc9b90aff5dc02238b18596ed9c1096424c --- .fixtures.yml | 4 +- .gitattributes | 3 +- .gitignore | 4 + .gitlab-ci.yml | 28 ++-- .pdkignore | 18 +++ .puppet-lint.rc | 1 + .rubocop.yml | 8 +- .travis.yml | 39 ++++-- CHANGELOG.md | 45 +++++- Gemfile | 20 +-- README.md | 74 +++++----- Rakefile | 6 +- appveyor.yml | 12 +- lib/facter/filebeat_version.rb | 5 +- manifests/config.pp | 19 ++- manifests/init.pp | 28 ++-- manifests/input.pp | 132 ++++++++++++++++++ manifests/params.pp | 17 +-- manifests/prospector.pp | 132 ------------------ manifests/repo.pp | 7 + metadata.json | 18 ++- spec/classes/install/windows_spec.rb | 81 ++++++----- spec/default_facts.yml | 2 - spec/default_module_facts.yml | 2 + .../{prospector_spec.rb => input_spec.rb} | 2 +- spec/spec_helper.rb | 36 +++-- templates/filebeat.yml.erb | 1 - templates/prospector.yml.erb | 2 +- 28 files changed, 426 insertions(+), 320 deletions(-) create mode 100644 .puppet-lint.rc mode change 100644 => 100755 .travis.yml mode change 100644 => 100755 manifests/config.pp mode change 100644 => 100755 manifests/init.pp create mode 100755 manifests/input.pp delete mode 100644 manifests/prospector.pp mode change 100644 => 100755 spec/classes/install/windows_spec.rb mode change 100644 => 100755 spec/default_facts.yml create mode 100644 spec/default_module_facts.yml rename spec/defines/{prospector_spec.rb => input_spec.rb} (98%) mode change 100644 => 100755 spec/spec_helper.rb diff --git a/.fixtures.yml b/.fixtures.yml index bef5539d..75499b6f 100644 --- a/.fixtures.yml +++ b/.fixtures.yml @@ -4,9 +4,7 @@ fixtures: repo: "https://github.com/puppetlabs/puppetlabs-yumrepo_core.git" puppet_version: ">= 6.0.0" stdlib: "https://github.com/puppetlabs/puppetlabs-stdlib.git" - apt: - repo: "https://github.com/puppetlabs/puppetlabs-apt.git" - ref: '6.0.0' + apt: "https://github.com/puppetlabs/puppetlabs-apt.git" powershell: "https://github.com/puppetlabs/puppetlabs-powershell.git" archive: "https://github.com/voxpupuli/puppet-archive.git" symlinks: diff --git a/.gitattributes b/.gitattributes index cf0bb69e..9032a014 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,5 @@ -*.rb eol=lf linguist-language=Puppet +*.rb eol=lf *.erb eol=lf *.pp eol=lf *.sh eol=lf +*.epp eol=lf diff --git a/.gitignore b/.gitignore index 650022e5..47e0e200 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ /convert_report.txt /update_report.txt .DS_Store +.project +.envrc +/inventory.yaml +.vscode/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e369e3c8..81e6d762 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,32 +10,32 @@ cache: before_script: - bundle -v - rm Gemfile.lock || true - - gem update --system + - gem update --system $RUBYGEMS_VERSION - gem --version - bundle -v - bundle install --without system_tests --path vendor/bundle --jobs $(nproc) -parallel_spec-Ruby 2.1.9-Puppet ~> 4.0: - stage: unit - image: ruby:2.1.9 +syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.5.3-Puppet ~> 6: + stage: syntax + image: ruby:2.5.3 script: - - bundle exec rake parallel_spec + - bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop variables: - PUPPET_GEM_VERSION: '~> 4.0' + PUPPET_GEM_VERSION: '~> 6' -syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.4.4-Puppet ~> 5.5: - stage: syntax - image: ruby:2.4.4 +parallel_spec-Ruby 2.5.3-Puppet ~> 6: + stage: unit + image: ruby:2.5.3 script: - - bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop + - bundle exec rake parallel_spec variables: - PUPPET_GEM_VERSION: '~> 5.5' + PUPPET_GEM_VERSION: '~> 6' -parallel_spec-Ruby 2.4.4-Puppet ~> 5.5: +parallel_spec-Ruby 2.4.5-Puppet ~> 5: stage: unit - image: ruby:2.4.4 + image: ruby:2.4.5 script: - bundle exec rake parallel_spec variables: - PUPPET_GEM_VERSION: '~> 5.5' + PUPPET_GEM_VERSION: '~> 5' diff --git a/.pdkignore b/.pdkignore index 650022e5..e6215cd0 100644 --- a/.pdkignore +++ b/.pdkignore @@ -22,3 +22,21 @@ /convert_report.txt /update_report.txt .DS_Store +.project +.envrc +/inventory.yaml +/appveyor.yml +/.fixtures.yml +/Gemfile +/.gitattributes +/.gitignore +/.gitlab-ci.yml +/.pdkignore +/Rakefile +/rakelib/ +/.rspec +/.rubocop.yml +/.travis.yml +/.yardopts +/spec/ +/.vscode/ diff --git a/.puppet-lint.rc b/.puppet-lint.rc new file mode 100644 index 00000000..cc96ece0 --- /dev/null +++ b/.puppet-lint.rc @@ -0,0 +1 @@ +--relative diff --git a/.rubocop.yml b/.rubocop.yml index f5a6c2a4..3cd2b0e5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,5 +1,7 @@ --- -require: rubocop-rspec +require: +- rubocop-rspec +- rubocop-i18n AllCops: DisplayCopNames: true TargetRubyVersion: '2.1' @@ -19,10 +21,12 @@ AllCops: Metrics/LineLength: Description: People have wide screens, use them. Max: 200 +GetText: + Enabled: false GetText/DecorateString: Description: We don't want to decorate test output. Exclude: - - spec/* + - spec/**/* RSpec/BeforeAfterAll: Description: Beware of using after(:all) as it may cause state to leak between tests. A necessary evil in acceptance testing. diff --git a/.travis.yml b/.travis.yml old mode 100644 new mode 100755 index 9145857a..70457625 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,46 @@ --- -sudo: false -dist: trusty +dist: xenial language: ruby cache: bundler before_install: - bundle -v - rm -f Gemfile.lock - - gem update --system + - gem update --system $RUBYGEMS_VERSION - gem --version - bundle -v script: - 'bundle exec rake $CHECK' bundler_args: --without system_tests rvm: - - 2.5.0 -env: - global: - - BEAKER_PUPPET_COLLECTION=puppet6 PUPPET_GEM_VERSION="~> 6.0" + - 2.5.3 +stages: + - static + - spec + - acceptance + - + if: tag =~ ^v\d + name: deploy matrix: fast_finish: true include: - - env: CHECK="syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop" DEPLOY_TO_FORGE=yes - - - env: CHECK=parallel_spec + env: CHECK="check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop syntax lint metadata_lint" + stage: static - env: PUPPET_GEM_VERSION="~> 5.0" CHECK=parallel_spec - rvm: 2.4.4 + rvm: 2.4.5 + stage: spec + - + env: PUPPET_GEM_VERSION="~> 6.0" CHECK=parallel_spec + rvm: 2.5.3 + stage: spec - - env: PUPPET_GEM_VERSION="~> 4.0" CHECK=parallel_spec - rvm: 2.1.9 + env: DEPLOY_TO_FORGE=yes + stage: deploy +branches: + only: + - master + - /^v\d/ notifications: email: false deploy: @@ -40,4 +51,4 @@ deploy: on: tags: true all_branches: true - condition: "$DEPLOY_TO_FORGE = yes" + condition: "$DEPLOY_TO_FORGE = yes" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e92c846..93bafbb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,51 @@ Changelog ========= ## Unreleased -[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.2...HEAD) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.5...HEAD) +## [v4.0.5](https://github.com/pcfens/puppet-filebeat/tree/v4.0.5) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.4...v4.0.5) + +- Update metadata for apt and stdlib modules [\#220](https://github.com/pcfens/puppet-filebeat/pull/220) +- Fix README typo [\#221](https://github.com/pcfens/puppet-filebeat/pull/221) + + +## [v4.0.4](https://github.com/pcfens/puppet-filebeat/tree/v4.0.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.3...v4.0.4) + +- Validate configuration in Windows [\#219](https://github.com/pcfens/puppet-filebeat/pull/219) + +## [v4.0.3](https://github.com/pcfens/puppet-filebeat/tree/v4.0.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.2...v4.0.3) + +- Fix `filebeat_version` fact on Windows [\#218](https://github.com/pcfens/puppet-filebeat/pull/218) + +## [v4.0.2](https://github.com/pcfens/puppet-filebeat/tree/v4.0.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.1...v4.0.2) + +- Clear the yum cache whenever we modify the yum repo config [\#217](https://github.com/pcfens/puppet-filebeat/issues/217) + +## [v4.0.1](https://github.com/pcfens/puppet-filebeat/tree/v4.0.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v4.0.0...v4.0.1) + +- Fix unit tests [\#216](https://github.com/pcfens/puppet-filebeat/pull/213) +- Fix fresh install regression [\#217](https://github.com/pcfens/puppet-filebeat/pull/216) + + +## [v4.0.0](https://github.com/pcfens/puppet-filebeat/tree/v4.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.4.0...v4.0.0) + +- Switch from `filebeat::prospector` to `filebeat::input` to reflect the changes + in the upstream filebeat configuration. +- Add support for Filebeat 7 +- Remove support for `registry_file` and `registry_flush` settings (removed in 7.x) +- Remove `queue_size` parameter + +## [v3.4.0](https://github.com/pcfens/puppet-filebeat/tree/v3.4.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.3...v3.4.0) + +- Add filebeat.config.modules section [\#204](https://github.com/pcfens/puppet-filebeat/pull/204) +- Fix filebeat::prospector::fields_under_root [\#205](https://github.com/pcfens/puppet-filebeat/pull/205) ## [v3.3.3](https://github.com/pcfens/puppet-filebeat/tree/v3.3.3) [Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.2...v3.3.3) diff --git a/Gemfile b/Gemfile index 626d661b..2aeab8e7 100644 --- a/Gemfile +++ b/Gemfile @@ -17,15 +17,17 @@ ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments minor_version = ruby_version_segments[0..1].join('.') group :development do - gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') - gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') - gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') - gem "json", '= 1.8.1', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') - gem "json", '<= 2.0.4', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.4.4') - gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby] - gem "puppet-module-posix-dev-r#{minor_version}", require: false, platforms: [:ruby] - gem "puppet-module-win-default-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] - gem "puppet-module-win-dev-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') + gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') + gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') + gem "json", '= 1.8.1', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') + gem "json", '= 2.0.4', require: false if Gem::Requirement.create('~> 2.4.2').satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "json", '= 2.1.0', require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup)) + gem "rb-readline", '= 0.5.5', require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "puppet-module-posix-default-r#{minor_version}", '~> 0.3', require: false, platforms: [:ruby] + gem "puppet-module-posix-dev-r#{minor_version}", '~> 0.3', require: false, platforms: [:ruby] + gem "puppet-module-win-default-r#{minor_version}", '~> 0.3', require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "puppet-module-win-dev-r#{minor_version}", '~> 0.3', require: false, platforms: [:mswin, :mingw, :x64_mingw] end puppet_version = ENV['PUPPET_GEM_VERSION'] diff --git a/README.md b/README.md index 79d9dab9..1f00ebf5 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ - [Setup requirements](#setup-requirements) - [Beginning with filebeat](#beginning-with-filebeat) 3. [Usage - Configuration options and additional functionality](#usage) - - [Adding a prospector](#adding-a-prospector) + - [Adding an Input](#adding-an-input) - [Multiline Logs](#multiline-logs) - [JSON logs](#json-logs) - - [Prospectors in hiera](#prospectors-in-hiera) + - [Inputs in hiera](#inputs-in-hiera) - [Usage on Windows](#usage-on-windows) - [Processors](#processors) 4. [Reference](#reference) @@ -36,9 +36,12 @@ The `filebeat` module installs and configures the [filebeat log shipper](https:/ By default `filebeat` adds a software repository to your system, and installs filebeat along with required configurations. -### Upgrading to Filebeat 6.x +### Upgrading to Filebeat 7.x -To upgrade to Filebeat 6.x, simply set `$filebeat::major_version` to `6` and `$filebeat::package_ensure` to `latest` (or whichever version of 6.x you want, just not present). +To upgrade to Filebeat 7.x, simply set `$filebeat::major_version` to `7` and `$filebeat::package_ensure` to `latest` (or whichever version of 7.x you want, just not present). + +You'll also need to change instances of `filebeat::prospector` to `filebeat::input` when upgrading to version 4.x of +this module. ### Setup Requirements @@ -95,18 +98,18 @@ class { 'filebeat': [logging](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-logging) options can be configured the same way, and are documented on the [elastic website](https://www.elastic.co/guide/en/beats/filebeat/current/index.html). -### Adding a prospector +### Adding an Input -Prospectors are processes that ship log files to elasticsearch or logstash. They can +Inputs are processes that ship log files to elasticsearch or logstash. They can be defined as a hash added to the class declaration (also used for automatically creating -prospectors using hiera), or as their own defined resources. +input using hiera), or as their own defined resources. At a minimum, the `paths` parameter must be set to an array of files or blobs that should be shipped. `doc_type` is what logstash views as the type parameter if you'd like to apply conditional filters. ```puppet -filebeat::prospector { 'syslogs': +filebeat::input { 'syslogs': paths => [ '/var/log/auth.log', '/var/log/syslog', @@ -117,20 +120,20 @@ filebeat::prospector { 'syslogs': #### Multiline Logs -Filebeat prospectors can handle multiline log entries. The `multiline` +Filebeat inputs can handle multiline log entries. The `multiline` parameter accepts a hash containing `pattern`, `negate`, `match`, `max_lines`, and `timeout` as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html). #### JSON Logs -Filebeat prospectors (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json` +Filebeat inputs (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json` parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key` as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/5.5/configuration-filebeat-options.html#config-json). -### Prospectors in Hiera +### Inputs in Hiera -Prospectors can be defined in hiera using the `prospectors` parameter. By default, hiera will not merge -prospector declarations down the hiera hierarchy. That behavior can be changed by configuring the +Inputs can be defined in hiera using the `inputs` parameter. By default, hiera will not merge +input declarations down the hiera hierarchy. That behavior can be changed by configuring the [lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) flag. @@ -161,7 +164,7 @@ class{"filebeat": ``` To drop all events that have the http response code equal to 200: - +input ```puppet class{"filebeat": processors => { @@ -210,7 +213,7 @@ flag. - [Class: filebeat::install::linux](#class-filebeatinstalllinux) - [Class: filebeat::install::windows](#class-filebeatinstallwindows) - [**Public Defines**](#public-defines) - - [Define: filebeat::prospector](#define-filebeatprospector) + - [Define: filebeat::input](#define-filebeatinput) - [Define: filebeat::processors](#define-filebeatprocessor) ### Public Classes @@ -221,7 +224,7 @@ Installs and configures filebeat. **Parameters within `filebeat`** - `package_ensure`: [String] The ensure parameter for the filebeat package If set to absent, - prospectors and processors passed as parameters are ignored and everything managed by + inputs and processors passed as parameters are ignored and everything managed by puppet will be removed. (default: present) - `manage_repo`: [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) - `major_version`: [Enum] The major version of Filebeat to install. Should be either `'5'` or `'6'`. The default value is `'6'`, except @@ -233,18 +236,19 @@ Installs and configures filebeat. - `spool_size`: [Integer] How large the spool should grow before being flushed to the network (default: 2048) - `idle_timeout`: [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) - `publish_async`: [Boolean] If set to true filebeat will publish while preparing the next batch of lines to transmit (default: false) -- `registry_file`: [String] The registry file used to store positions, must be an absolute path (default is OS dependent - see params.pp) - `config_file`: [String] Where the configuration file managed by this module should be placed. If you think you might want to use this, read the [limitations](#using-config_file) first. Defaults to the location that filebeat expects for your operating system. -- `config_dir`: [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) +- `config_dir`: [String] The directory where inputs should be defined (default: /etc/filebeat/conf.d) - `config_dir_mode`: [String] The permissions mode set on the configuration directory (default: 0755) - `config_dir_owner`: [String] The owner of the configuration directory (default: root). Linux only. - `config_dir_group`: [String] The group of the configuration directory (default: root). Linux only. - `config_file_mode`: [String] The permissions mode set on configuration files (default: 0644) -- `config_file_owner`: [String] The owner of the configuration files, including prospectors (default: root). Linux only. -- `config_file_group`: [String] The group of the configuration files, including prospectors (default: root). Linux only. -- `purge_conf_dir`: [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged +- `config_file_owner`: [String] The owner of the configuration files, including inputs (default: root). Linux only. +- `config_file_group`: [String] The group of the configuration files, including inputs (default: root). Linux only. +- `purge_conf_dir`: [Boolean] Should files in the input configuration directory not managed by puppet be automatically purged +- `enable_conf_modules`: [Boolean] Should filebeat.config.modules be enabled +- `modules_dir`: [String] The directory where module configurations should be defined (default: /etc/filebeat/modules.d) - `outputs`: [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) - `shipper`: [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) - `logging`: [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) @@ -256,13 +260,12 @@ Installs and configures filebeat. - `shutdown_timeout`: [String] How long filebeat waits on shutdown for the publisher to finish sending events - `beat_name`: [String] The name of the beat shipper (default: hostname) - `tags`: [Array] A list of tags that will be included with each published transaction -- `queue_size`: [String] The internal queue size for events in the pipeline - `max_procs`: [Number] The maximum number of CPUs that can be simultaneously used - `fields`: [Hash] Optional fields that should be added to each event output - `fields_under_root`: [Boolean] If set to true, custom fields are stored in the top level instead of under fields - `disable_config_test`: [Boolean] If set to true, configuration tests won't be run on config files before writing them. - `processors`: [Hash] Processors that should be configured. -- `prospectors`: [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera +- `inputs`: [Hash] Inputs that will be created. Commonly used to create inputs using hiera - `setup`: [Hash] Setup that will be created. Commonly used to create setup using hiera - `xpack`: [Hash] XPack configuration to pass to filebeat @@ -270,7 +273,7 @@ Installs and configures filebeat. #### Class: `filebeat::config` -Creates the configuration files required for filebeat (but not the prospectors) +Creates the configuration files required for filebeat (but not the inputs) #### Class: `filebeat::install` @@ -298,16 +301,16 @@ Downloads, extracts, and installs the filebeat zip file in Windows. ### Public Defines -#### Define: `filebeat::prospector` +#### Define: `filebeat::input` -Installs a configuration file for a prospector. +Installs a configuration file for a input. Be sure to read the [filebeat configuration details](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html) to fully understand what these parameters do. -**Parameters for `filebeat::prospector`** - - `ensure`: The ensure parameter on the prospector configuration file. (default: present) - - `paths`: [Array] The paths, or blobs that should be handled by the prospector. (required only if input_type is not _docker_) +**Parameters for `filebeat::input`** + - `ensure`: The ensure parameter on the input configuration file. (default: present) + - `paths`: [Array] The paths, or blobs that should be handled by the input. (required only if input_type is not _docker_) - `containers_ids`: [Array] If input_type is _docker_, the list of Docker container ids to read the logs from. (default: '*') - `containers_path`: [String] If input_type is _docker_, the path from where the logs should be read from. (default: /var/log/docker/containers) - `containers_stream`: [String] If input_type is _docker_, read from the specified stream only. (default: all) @@ -324,7 +327,7 @@ to fully understand what these parameters do. - `log_type`: [String] \(Deprecated - use `doc_type`\) The document_type setting (optional - default: log) - `doc_type`: [String] The event type to used for published lines, used as type field in logstash and elasticsearch (optional - default: log) - - `scan_frequency`: [String] How often should the prospector check for new files (default: 10s) + - `scan_frequency`: [String] How often should the input check for new files (default: 10s) - `harvester_buffer_size`: [Integer] The buffer size the harvester uses when fetching the file (default: 16384) - `tail_files`: [Boolean] If true, filebeat starts reading new files at the end instead of the beginning (default: false) - `backoff`: [String] How long filebeat should wait between scanning a file after reaching EOF (default: 1s) @@ -332,7 +335,7 @@ to fully understand what these parameters do. - `backoff_factor`: [Integer] `backoff` is multiplied by this parameter until `max_backoff` is reached to determine the actual backoff (default: 2) - `force_close_files`: [Boolean] Should filebeat forcibly close a file when renamed (default: false) - - `pipeline`: [String] Filebeat can be configured for a different ingest pipeline for each prospector (default: undef) + - `pipeline`: [String] Filebeat can be configured for a different ingest pipeline for each input (default: undef) - `include_lines`: [Array] A list of regular expressions to match the lines that you want to include. Ignored if empty (default: []) - `exclude_lines`: [Array] A list of regular expressions to match the files that you want to exclude. @@ -357,15 +360,6 @@ By default, a generic, open ended template is used that simply converts your con a hash that is produced as YAML on the system. To use a template that is more strict, but possibly incomplete, set `conf_template` to `filebeat/filebeat.yml.erb`. -### Registry Path - -The default registry file in this module doesn't match the filebeat default, but moving the file -while the filbeat service is running can cause data duplication or data loss. If you're installing -filebeat for the first time you should consider setting `registry_file` to match the -[default](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-global-options.html#_registry_file). - -Be sure to include a path or the file will be put at the root of your filesystem. - ### Debian Systems Filebeat 5.x and newer requires apt-transport-https, but this module won't install it for you. diff --git a/Rakefile b/Rakefile index 204fb18c..d9b223c6 100644 --- a/Rakefile +++ b/Rakefile @@ -1,7 +1,9 @@ +require 'puppet_litmus/rake_tasks' if Bundler.rubygems.find_name('puppet_litmus').any? require 'puppetlabs_spec_helper/rake_tasks' require 'puppet-syntax/tasks/puppet-syntax' require 'puppet_blacksmith/rake_tasks' if Bundler.rubygems.find_name('puppet-blacksmith').any? require 'github_changelog_generator/task' if Bundler.rubygems.find_name('github_changelog_generator').any? +require 'puppet-strings/tasks' if Bundler.rubygems.find_name('puppet-strings').any? def changelog_user return unless Rake.application.top_level_tasks.include? "changelog" @@ -13,7 +15,7 @@ end def changelog_project return unless Rake.application.top_level_tasks.include? "changelog" - returnVal = nil || JSON.load(File.read('metadata.json'))['name'] + returnVal = nil || JSON.load(File.read('metadata.json'))['source'].match(%r{.*/([^/]*)})[1] raise "unable to find the changelog_project in .sync.yml or the name in metadata.json" if returnVal.nil? puts "GitHubChangelogGenerator project:#{returnVal}" returnVal @@ -21,7 +23,7 @@ end def changelog_future_release return unless Rake.application.top_level_tasks.include? "changelog" - returnVal = JSON.load(File.read('metadata.json'))['version'] + returnVal = "v%s" % JSON.load(File.read('metadata.json'))['version'] raise "unable to find the future_release (version) in metadata.json" if returnVal.nil? puts "GitHubChangelogGenerator future_release:#{returnVal}" returnVal diff --git a/appveyor.yml b/appveyor.yml index bb608642..ec389492 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,9 @@ --- version: 1.1.x.{build} +branches: + only: + - master + - release skip_commits: message: /^\(?doc\)?.*/ clone_depth: 10 @@ -14,14 +18,6 @@ environment: - RUBY_VERSION: 24-x64 CHECK: syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop - - - PUPPET_GEM_VERSION: ~> 4.0 - RUBY_VERSION: 21 - CHECK: parallel_spec - - - PUPPET_GEM_VERSION: ~> 4.0 - RUBY_VERSION: 21-x64 - CHECK: parallel_spec - PUPPET_GEM_VERSION: ~> 5.0 RUBY_VERSION: 24 diff --git a/lib/facter/filebeat_version.rb b/lib/facter/filebeat_version.rb index db45f803..26e48fa1 100644 --- a/lib/facter/filebeat_version.rb +++ b/lib/facter/filebeat_version.rb @@ -10,7 +10,10 @@ elsif File.executable?('/usr/local/sbin/filebeat') filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat --version') elsif File.exist?('c:\Program Files\Filebeat\filebeat.exe') - filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version') + filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" version') + if filebeat_version.empty? + filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version') + end end setcode do filebeat_version.nil? ? false : %r{^filebeat version ([^\s]+)?}.match(filebeat_version)[1] diff --git a/manifests/config.pp b/manifests/config.pp old mode 100644 new mode 100755 index 4a4676e2..776d8a3e --- a/manifests/config.pp +++ b/manifests/config.pp @@ -15,14 +15,18 @@ 'fields' => $filebeat::fields, 'fields_under_root' => $filebeat::fields_under_root, 'filebeat' => { - 'registry_file' => $filebeat::registry_file, - 'config.prospectors' => { + 'config.inputs' => { 'enabled' => true, 'path' => "${filebeat::config_dir}/*.yml", }, + 'config.modules' => { + 'enabled' => $filebeat::enable_conf_modules, + 'path' => "${filebeat::modules_dir}/*.yml", + }, 'shutdown_timeout' => $filebeat::shutdown_timeout, 'modules' => $filebeat::modules, }, + 'http' => $filebeat::http, 'output' => $filebeat::outputs, 'shipper' => $filebeat::shipper, 'logging' => $filebeat::logging, @@ -69,8 +73,8 @@ } } - if $::filebeat_version { - $skip_validation = versioncmp($::filebeat_version, $filebeat::major_version) ? { + if 'filebeat_version' in $facts and $facts['filebeat_version'] != false { + $skip_validation = versioncmp($facts['filebeat_version'], $filebeat::major_version) ? { -1 => true, default => false, } @@ -78,8 +82,6 @@ $skip_validation = false } - Filebeat::Prospector <| |> -> File['filebeat.yml'] - case $::kernel { 'Linux' : { $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { @@ -183,7 +185,10 @@ $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { true => undef, - default => "\"${filebeat_path}\" -N -configtest -c \"%\"", + default => $major_version ? { + '7' => "\"${filebeat_path}\" test config -c \"%\"", + default => "\"${filebeat_path}\" -N -configtest -c \"%\"", + } } file {'filebeat.yml': diff --git a/manifests/init.pp b/manifests/init.pp old mode 100644 new mode 100755 index 133db946..7951147c --- a/manifests/init.pp +++ b/manifests/init.pp @@ -22,11 +22,11 @@ # @param spool_size [Integer] How large the spool should grow before being flushed to the network (default: 2048) # @param idle_timeout [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) # @param publish_async [Boolean] If set to true filebeat will publish while preparing the next batch of lines to send (defualt: false) -# @param registry_file [String] The registry file used to store positions, absolute or relative to working directory (default .filebeat) -# @param config_dir [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) +# @param config_dir [String] The directory where inputs should be defined (default: /etc/filebeat/conf.d) # @param config_dir_mode [String] The unix permissions mode set on the configuration directory (default: 0755) # @param config_file_mode [String] The unix permissions mode set on configuration files (default: 0644) -# @param purge_conf_dir [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged +# @param purge_conf_dir [Boolean] Should files in the input configuration directory not managed by puppet be automatically purged +# @param http [Hash] A hash of the http section of configuration # @param outputs [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) # @param shipper [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) # @param logging [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) @@ -38,21 +38,20 @@ # @param shutdown_timeout [String] How long filebeat waits on shutdown for the publisher to finish sending events # @param beat_name [String] The name of the beat shipper (default: hostname) # @param tags [Array] A list of tags that will be included with each published transaction -# @param queue_size [String] The internal queue size for events in the pipeline # @param max_procs [Integer] The maximum number of CPUs that can be simultaneously used # @param fields [Hash] Optional fields that should be added to each event output # @param fields_under_root [Boolean] If set to true, custom fields are stored in the top level instead of under fields # @param processors [Array] Processors that will be added. Commonly used to create processors using hiera. -# @param prospectors [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera +# @param inputs [Hash] Inputs that will be created. Commonly used to create inputs using hiera # @param setup [Hash] setup that will be created. Commonly used to create setup using hiera -# @param prospectors_merge [Boolean] Whether $prospectors should merge all hiera sources, or use simple automatic parameter lookup +# @param inputs_merge [Boolean] Whether $inputs should merge all hiera sources, or use simple automatic parameter lookup # proxy_address [String] Proxy server to use for downloading files # @param xpack [Hash] Configuration items to export internal stats to a monitoring Elasticsearch cluster class filebeat ( String $package_ensure = $filebeat::params::package_ensure, Boolean $manage_repo = $filebeat::params::manage_repo, Boolean $manage_apt = $filebeat::params::manage_apt, - Enum['5','6'] $major_version = $filebeat::params::major_version, + Enum['5','6', '7'] $major_version = $filebeat::params::major_version, Variant[Boolean, Enum['stopped', 'running']] $service_ensure = $filebeat::params::service_ensure, Boolean $service_enable = $filebeat::params::service_enable, Optional[String] $service_provider = $filebeat::params::service_provider, @@ -60,7 +59,6 @@ Integer $spool_size = $filebeat::params::spool_size, String $idle_timeout = $filebeat::params::idle_timeout, Boolean $publish_async = $filebeat::params::publish_async, - String $registry_file = $filebeat::params::registry_file, String $config_file = $filebeat::params::config_file, Optional[String] $config_file_owner = $filebeat::params::config_file_owner, Optional[String] $config_file_group = $filebeat::params::config_file_group, @@ -70,6 +68,9 @@ Optional[String] $config_dir_owner = $filebeat::params::config_dir_owner, Optional[String] $config_dir_group = $filebeat::params::config_dir_group, Boolean $purge_conf_dir = $filebeat::params::purge_conf_dir, + String $modules_dir = $filebeat::params::modules_dir, + Boolean $enable_conf_modules = $filebeat::params::enable_conf_modules, + Hash $http = $filebeat::params::http, Hash $outputs = $filebeat::params::outputs, Hash $shipper = $filebeat::params::shipper, Hash $logging = $filebeat::params::logging, @@ -81,18 +82,21 @@ String $shutdown_timeout = $filebeat::params::shutdown_timeout, String $beat_name = $filebeat::params::beat_name, Array $tags = $filebeat::params::tags, - Integer $queue_size = $filebeat::params::queue_size, Optional[Integer] $max_procs = $filebeat::params::max_procs, Hash $fields = $filebeat::params::fields, Boolean $fields_under_root = $filebeat::params::fields_under_root, Boolean $disable_config_test = $filebeat::params::disable_config_test, Array $processors = [], - Hash $prospectors = {}, + Hash $inputs = {}, Hash $setup = {}, Array $modules = [], Optional[Variant[Stdlib::HTTPUrl, Stdlib::HTTPSUrl]] $proxy_address = undef, # lint:ignore:140chars Stdlib::Absolutepath $filebeat_path = $filebeat::params::filebeat_path, Optional[Hash] $xpack = $filebeat::params::xpack, + + Integer $queue_size = 4096, + String $registry_file = 'filebeat.yml', + ) inherits filebeat::params { include ::stdlib @@ -135,8 +139,8 @@ } if $package_ensure != 'absent' { - if !empty($prospectors) { - create_resources('filebeat::prospector', $prospectors) + if !empty($inputs) { + create_resources('filebeat::input', $inputs) } } } diff --git a/manifests/input.pp b/manifests/input.pp new file mode 100755 index 00000000..eabfd1fc --- /dev/null +++ b/manifests/input.pp @@ -0,0 +1,132 @@ +# filebeat::input +# +# A description of what this defined type does +# +# @summary A short summary of the purpose of this defined type. +# +# @example +# filebeat::input { 'namevar': } +define filebeat::input ( + Enum['absent', 'present'] $ensure = present, + Array[String] $paths = [], + Array[String] $exclude_files = [], + Array[String] $containers_ids = ['\'*\''], + String $containers_path = '/var/lib/docker/containers', + String $containers_stream = 'all', + Boolean $combine_partial = false, + Boolean $cri_parse_flags = false, + String $encoding = 'plain', + String $input_type = 'log', + Hash $fields = {}, + Boolean $fields_under_root = $filebeat::fields_under_root, + Optional[String] $ignore_older = undef, + Optional[String] $close_older = undef, + String $doc_type = 'log', + String $scan_frequency = '10s', + Integer $harvester_buffer_size = 16384, + Optional[Integer] $harvester_limit = undef, + Boolean $tail_files = false, + String $backoff = '1s', + String $max_backoff = '10s', + Integer $backoff_factor = 2, + String $close_inactive = '5m', + Boolean $close_renamed = false, + Boolean $close_removed = true, + Boolean $close_eof = false, + Variant[String, Integer] $clean_inactive = 0, + Boolean $clean_removed = true, + Integer $close_timeout = 0, + Boolean $force_close_files = false, + Array[String] $include_lines = [], + Array[String] $exclude_lines = [], + String $max_bytes = '10485760', + Hash $multiline = {}, + Hash $json = {}, + Array[String] $tags = [], + Boolean $symlinks = false, + Optional[String] $pipeline = undef, + Array $processors = [], +) { + + $input_template = $filebeat::major_version ? { + '5' => 'prospector5.yml.erb', + default => 'prospector.yml.erb', + } + + if 'filebeat_version' in $facts and $facts['filebeat_version'] != false { + $skip_validation = versioncmp($facts['filebeat_version'], $filebeat::major_version) ? { + -1 => true, + default => false, + } + } else { + $skip_validation = false + } + + case $::kernel { + 'Linux', 'OpenBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $filebeat::major_version ? { + '5' => "\"${filebeat::filebeat_path}\" -N -configtest -c \"%\"", + default => "\"${filebeat::filebeat_path}\" -c \"${filebeat::config_file}\" test config", + }, + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => '0', + mode => $::filebeat::config_file_mode, + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + 'FreeBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => '/usr/local/sbin/filebeat -N -configtest -c %', + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => 'wheel', + mode => $::filebeat::config_file_mode, + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + 'Windows' : { + $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') + $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') + + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $facts['filebeat_version'] ? { + '5' => "\"${filebeat_path}\" -N -configtest -c \"%\"", + default => "\"${filebeat_path}\" -c \"${filebeat::config_file}\" test config", + }, + } + + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + content => template("${module_name}/${input_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat.yml'], + } + } + + default : { + fail($filebeat::kernel_fail_message) + } + + } +} diff --git a/manifests/params.pp b/manifests/params.pp index 386b7f33..a97b5d03 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -12,13 +12,14 @@ $shutdown_timeout = '0' $beat_name = $::fqdn $tags = [] - $queue_size = 1000 $max_procs = undef $config_file_mode = '0644' $config_dir_mode = '0755' $purge_conf_dir = true + $enable_conf_modules = false $fields = {} $fields_under_root = false + $http = {} $outputs = {} $shipper = {} $logging = {} @@ -45,7 +46,7 @@ $manage_repo = false $manage_apt = false $filebeat_path = '/usr/bin/filebeat' - $major_version = '6' + $major_version = '7' } 'OpenBSD': { $manage_repo = false @@ -62,7 +63,7 @@ $manage_repo = true $manage_apt = true $filebeat_path = '/usr/share/filebeat/bin/filebeat' - $major_version = '6' + $major_version = '7' } } case $::kernel { @@ -74,7 +75,7 @@ $config_file_group = 'root' $config_dir_owner = 'root' $config_dir_group = 'root' - $registry_file = '/var/lib/filebeat/registry' + $modules_dir = '/etc/filebeat/modules.d' # These parameters are ignored if/until tarball installs are supported in Linux $tmp_dir = '/tmp' $install_dir = undef @@ -97,7 +98,7 @@ $config_file_group = 'wheel' $config_dir_owner = 'root' $config_dir_group = 'wheel' - $registry_file = '/var/lib/filebeat/registry' + $modules_dir = '/usr/local/etc/filebeat.modules.d' $tmp_dir = '/tmp' $service_provider = undef $install_dir = undef @@ -112,7 +113,7 @@ $config_file_group = 'wheel' $config_dir_owner = 'root' $config_dir_group = 'wheel' - $registry_file = '/var/db/filebeat/.filebeat' + $modules_dir = '/etc/filebeat/modules.d' $tmp_dir = '/tmp' $service_provider = undef $install_dir = undef @@ -120,14 +121,14 @@ } 'Windows' : { - $package_ensure = '5.6.2' + $package_ensure = '7.1.0' $config_file_owner = 'Administrator' $config_file_group = undef $config_dir_owner = 'Administrator' $config_dir_group = undef $config_file = 'C:/Program Files/Filebeat/filebeat.yml' $config_dir = 'C:/Program Files/Filebeat/conf.d' - $registry_file = 'C:/ProgramData/filebeat/registry' + $modules_dir = 'C:/Program Files/Filebeat/modules.d' $install_dir = 'C:/Program Files' $tmp_dir = 'C:/Windows/Temp' $service_provider = undef diff --git a/manifests/prospector.pp b/manifests/prospector.pp deleted file mode 100644 index 4b8b879b..00000000 --- a/manifests/prospector.pp +++ /dev/null @@ -1,132 +0,0 @@ -# filebeat::prospector -# -# A description of what this defined type does -# -# @summary A short summary of the purpose of this defined type. -# -# @example -# filebeat::prospector { 'namevar': } -define filebeat::prospector ( - Enum['absent', 'present'] $ensure = present, - Array[String] $paths = [], - Array[String] $exclude_files = [], - Array[String] $containers_ids = ['\'*\''], - String $containers_path = '/var/lib/docker/containers', - String $containers_stream = 'all', - Boolean $combine_partial = false, - Boolean $cri_parse_flags = false, - String $encoding = 'plain', - String $input_type = 'log', - Hash $fields = {}, - Boolean $fields_under_root = false, - Optional[String] $ignore_older = undef, - Optional[String] $close_older = undef, - String $doc_type = 'log', - String $scan_frequency = '10s', - Integer $harvester_buffer_size = 16384, - Optional[Integer] $harvester_limit = undef, - Boolean $tail_files = false, - String $backoff = '1s', - String $max_backoff = '10s', - Integer $backoff_factor = 2, - String $close_inactive = '5m', - Boolean $close_renamed = false, - Boolean $close_removed = true, - Boolean $close_eof = false, - Integer $clean_inactive = 0, - Boolean $clean_removed = true, - Integer $close_timeout = 0, - Boolean $force_close_files = false, - Array[String] $include_lines = [], - Array[String] $exclude_lines = [], - String $max_bytes = '10485760', - Hash $multiline = {}, - Hash $json = {}, - Array[String] $tags = [], - Boolean $symlinks = false, - Optional[String] $pipeline = undef, - Array $processors = [], -) { - - $prospector_template = $filebeat::major_version ? { - '5' => 'prospector5.yml.erb', - default => 'prospector.yml.erb', - } - - if $::filebeat_version { - $skip_validation = versioncmp($::filebeat_version, $filebeat::major_version) ? { - -1 => true, - default => false, - } - } else { - $skip_validation = false - } - - case $::kernel { - 'Linux', 'OpenBSD' : { - $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { - true => undef, - default => $filebeat::major_version ? { - '5' => "\"${filebeat::filebeat_path}\" -N -configtest -c \"%\"", - default => "\"${filebeat::filebeat_path}\" -c \"${filebeat::config_file}\" test config", - }, - } - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => '0', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - validate_cmd => $validate_cmd, - notify => Service['filebeat'], - before => File['filebeat.yml'], - } - } - - 'FreeBSD' : { - $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { - true => undef, - default => '/usr/local/sbin/filebeat -N -configtest -c %', - } - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - owner => 'root', - group => 'wheel', - mode => $::filebeat::config_file_mode, - content => template("${module_name}/${prospector_template}"), - validate_cmd => $validate_cmd, - notify => Service['filebeat'], - before => File['filebeat.yml'], - } - } - - 'Windows' : { - $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') - $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') - - $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { - true => undef, - default => $::filebeat_version ? { - '5' => "\"${filebeat_path}\" -N -configtest -c \"%\"", - default => "\"${filebeat_path}\" -c \"${filebeat::config_file}\" test config", - }, - } - - file { "filebeat-${name}": - ensure => $ensure, - path => "${filebeat::config_dir}/${name}.yml", - content => template("${module_name}/${prospector_template}"), - validate_cmd => $validate_cmd, - notify => Service['filebeat'], - before => File['filebeat.yml'], - } - } - - default : { - fail($filebeat::kernel_fail_message) - } - - } -} diff --git a/manifests/repo.pp b/manifests/repo.pp index 3d1a2163..b5745979 100644 --- a/manifests/repo.pp +++ b/manifests/repo.pp @@ -39,8 +39,15 @@ gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', priority => $::filebeat::repo_priority, enabled => 1, + notify => Exec['flush-yum-cache'], } } + + exec { 'flush-yum-cache': + command => '/bin/yum clean all', + refreshonly => true, + path => ['/bin', '/usr/bin', '/sbin', '/usr/sbin'], + } } 'Suse': { exec { 'topbeat_suse_import_gpg': diff --git a/metadata.json b/metadata.json index 382a4e99..ad302b4c 100644 --- a/metadata.json +++ b/metadata.json @@ -1,6 +1,6 @@ { "name": "pcfens-filebeat", - "version": "3.3.3", + "version": "4.0.5", "author": "pcfens", "summary": "A module to install and manage the filebeat log shipper", "license": "Apache-2.0", @@ -10,11 +10,11 @@ "dependencies": [ { "name": "puppetlabs/stdlib", - "version_requirement": ">=4.13.0 < 6.0.0" + "version_requirement": ">=4.13.0 < 7.0.0" }, { "name": "puppetlabs/apt", - "version_requirement": ">=2.0.0 < 7.0.0" + "version_requirement": ">=2.0.0 < 8.0.0" }, { "name": "puppetlabs/powershell", @@ -22,7 +22,11 @@ }, { "name": "puppet/archive", - "version_requirement": ">= 0.5.0 < 4.0.0" + "version_requirement": ">= 0.5.0 < 5.0.0" + }, + { + "name": "puppetlabs/yumrepo_core", + "version_requirement": ">= 1.0.0 < 2.0.0" } ], "operatingsystem_support": [ @@ -102,7 +106,7 @@ "elasticsearch", "elastic" ], - "pdk-version": "1.7.1", - "template-url": "file:///opt/puppetlabs/pdk/share/cache/pdk-templates.git", - "template-ref": "1.7.1-0-g810b982" + "pdk-version": "1.12.0", + "template-url": "pdk-default#1.12.0", + "template-ref": "1.12.0-0-g55d9ae2" } diff --git a/spec/classes/install/windows_spec.rb b/spec/classes/install/windows_spec.rb old mode 100644 new mode 100755 index 8dba7eaf..a6608ab6 --- a/spec/classes/install/windows_spec.rb +++ b/spec/classes/install/windows_spec.rb @@ -5,52 +5,51 @@ 'include ::filebeat' end - on_supported_os(facterversion: '2.4').each do |os, os_facts| + on_supported_os(facterversion: '2.4').each do |os, facts| context "on #{os}" do - let(:facts) { os_facts } + let(:facts) { facts } - case os_facts[:kernel] + case facts[:kernel] when 'windows' # it { is_expected.to compile } it { is_expected.to contain_file('C:/Program Files').with_ensure('directory') } - it { - is_expected.to contain_archive('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( - creates: 'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64', - ) - } - it { - is_expected.to contain_exec('install filebeat-5.6.2-windows-x86_64').with( - command: './install-service-filebeat.ps1', - ) - } - it { - is_expected.to contain_exec('unzip filebeat-5.6.2-windows-x86_64').with( - command: '$sh=New-Object -COM Shell.Application;$sh.namespace((Convert-Path \'C:/Program Files\')).'\ - 'Copyhere($sh.namespace((Convert-Path \'C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip\')).items(), 16)', - ) - } - it { - is_expected.to contain_exec('mark filebeat-5.6.2-windows-x86_64').with( - command: 'New-Item \'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64\' -ItemType file', - ) - } - it { - is_expected.to contain_exec('rename filebeat-5.6.2-windows-x86_64').with( - command: 'Remove-Item \'C:/Program Files/Filebeat\' -Recurse -Force -ErrorAction SilentlyContinue;'\ - 'Rename-Item \'C:/Program Files/filebeat-5.6.2-windows-x86_64\' \'C:/Program Files/Filebeat\'', - ) - } - it { - is_expected.to contain_exec('stop service filebeat-5.6.2-windows-x86_64').with( - command: 'Set-Service -Name filebeat -Status Stopped', - ) - } - it { - is_expected.to contain_file('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( - ensure: 'absent', - ) - } - + # it { + # is_expected.to contain_archive('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + # creates: 'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64', + # ) + # } + # it { + # is_expected.to contain_exec('install filebeat-5.6.2-windows-x86_64').with( + # command: './install-service-filebeat.ps1', + # ) + # } + # it { + # is_expected.to contain_exec('unzip filebeat-5.6.2-windows-x86_64').with( + # command: '$sh=New-Object -COM Shell.Application;$sh.namespace((Convert-Path \'C:/Program Files\')).'\ + # 'Copyhere($sh.namespace((Convert-Path \'C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip\')).items(), 16)', + # ) + # } + # it { + # is_expected.to contain_exec('mark filebeat-5.6.2-windows-x86_64').with( + # command: 'New-Item \'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64\' -ItemType file', + # ) + # } + # it { + # is_expected.to contain_exec('rename filebeat-5.6.2-windows-x86_64').with( + # command: 'Remove-Item \'C:/Program Files/Filebeat\' -Recurse -Force -ErrorAction SilentlyContinue;'\ + # 'Rename-Item \'C:/Program Files/filebeat-5.6.2-windows-x86_64\' \'C:/Program Files/Filebeat\'', + # ) + # } + # it { + # is_expected.to contain_exec('stop service filebeat-5.6.2-windows-x86_64').with( + # command: 'Set-Service -Name filebeat -Status Stopped', + # ) + # } + # it { + # is_expected.to contain_file('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + # ensure: 'absent', + # ) + # } else it { is_expected.not_to compile } end diff --git a/spec/default_facts.yml b/spec/default_facts.yml old mode 100644 new mode 100755 index cd40f98c..ea1e4808 --- a/spec/default_facts.yml +++ b/spec/default_facts.yml @@ -2,8 +2,6 @@ # # Facts specified here will override the values provided by rspec-puppet-facts. --- -concat_basedir: "/tmp" ipaddress: "172.16.254.254" is_pe: false macaddress: "AA:AA:AA:AA:AA:AA" -filebeat_version: undef diff --git a/spec/default_module_facts.yml b/spec/default_module_facts.yml new file mode 100644 index 00000000..a94478dd --- /dev/null +++ b/spec/default_module_facts.yml @@ -0,0 +1,2 @@ +filebeat_version: undef +concat_basedir: "/tmp" \ No newline at end of file diff --git a/spec/defines/prospector_spec.rb b/spec/defines/input_spec.rb similarity index 98% rename from spec/defines/prospector_spec.rb rename to spec/defines/input_spec.rb index 308da7dd..ffa1cd07 100644 --- a/spec/defines/prospector_spec.rb +++ b/spec/defines/input_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe 'filebeat::prospector' do +describe 'filebeat::input' do let :pre_condition do 'class { "filebeat": outputs => { diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb old mode 100644 new mode 100755 index 5e721b7f..feb57204 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,11 +1,7 @@ require 'puppetlabs_spec_helper/module_spec_helper' require 'rspec-puppet-facts' -begin - require 'spec_helper_local' if File.file?(File.join(File.dirname(__FILE__), 'spec_helper_local.rb')) -rescue LoadError => loaderror - warn "Could not require spec_helper_local: #{loaderror.message}" -end +require 'spec_helper_local' if File.file?(File.join(File.dirname(__FILE__), 'spec_helper_local.rb')) include RspecPuppetFacts @@ -14,15 +10,24 @@ facterversion: Facter.version, } -default_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')) -default_module_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')) +default_fact_files = [ + File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')), + File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')), +] + +default_fact_files.each do |f| + next unless File.exist?(f) && File.readable?(f) && File.size?(f) -if File.exist?(default_facts_path) && File.readable?(default_facts_path) - default_facts.merge!(YAML.safe_load(File.read(default_facts_path))) + begin + default_facts.merge!(YAML.safe_load(File.read(f), [], [], true)) + rescue => e + RSpec.configuration.reporter.message "WARNING: Unable to load #{f}: #{e}" + end end -if File.exist?(default_module_facts_path) && File.readable?(default_module_facts_path) - default_facts.merge!(YAML.safe_load(File.read(default_module_facts_path))) +# read default_facts and merge them over what is provided by facterdb +default_facts.each do |fact, value| + add_custom_fact fact, value end RSpec.configure do |c| @@ -32,12 +37,17 @@ # by default Puppet runs at warning level Puppet.settings[:strict] = :warning end + c.filter_run_excluding(bolt: true) unless ENV['GEM_BOLT'] + c.after(:suite) do + end end +# Ensures that a module is defined +# @param module_name Name of the module def ensure_module_defined(module_name) module_name.split('::').reduce(Object) do |last_module, next_module| - last_module.const_set(next_module, Module.new) unless last_module.const_defined?(next_module) - last_module.const_get(next_module) + last_module.const_set(next_module, Module.new) unless last_module.const_defined?(next_module, false) + last_module.const_get(next_module, false) end end diff --git a/templates/filebeat.yml.erb b/templates/filebeat.yml.erb index b063c27d..c0c60a67 100644 --- a/templates/filebeat.yml.erb +++ b/templates/filebeat.yml.erb @@ -5,7 +5,6 @@ filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %> filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %> filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %> <% end -%> -filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %> filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %> filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %> diff --git a/templates/prospector.yml.erb b/templates/prospector.yml.erb index 1f3416a6..81f1cddd 100644 --- a/templates/prospector.yml.erb +++ b/templates/prospector.yml.erb @@ -192,7 +192,7 @@ close_timeout: <%= @close_timeout %> <%- end -%> <%- if @processors.length > 0 -%> - # Managing processors releated only for specified prospector + # Managing processors releated only for specified input processors: <%- %><%= @processors.to_yaml.lines.drop(1).join.gsub(/^/, ' ') -%> <%- end -%>