From 8f6eeacf55db4ad61a62035ce0c83c1b021ca411 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 18 Jan 2019 09:42:31 +0100 Subject: [PATCH] Squashed '.puppet/modules/filebeat/' content from commit 351db2815 git-subtree-dir: .puppet/modules/filebeat git-subtree-split: 351db28155ab5c0eb3797b6cbb32f0a0768ab367 --- .fixtures.yml | 13 + .gitattributes | 4 + .gitignore | 24 + .gitlab-ci.yml | 41 ++ .pdkignore | 24 + .pmtignore | 20 + .rspec | 2 + .rubocop.yml | 122 +++++ .travis.yml | 43 ++ .yardopts | 1 + CHANGELOG.md | 431 +++++++++++++++ Gemfile | 70 +++ LICENSE | 202 +++++++ NOTICE | 13 + README.md | 395 ++++++++++++++ Rakefile | 75 +++ appveyor.yml | 60 +++ lib/facter/filebeat_version.rb | 18 + manifests/config.pp | 211 ++++++++ manifests/init.pp | 142 +++++ manifests/install.pp | 43 ++ manifests/install/freebsd.pp | 13 + manifests/install/linux.pp | 15 + manifests/install/openbsd.pp | 6 + manifests/install/windows.pp | 82 +++ manifests/params.pp | 145 +++++ manifests/prospector.pp | 132 +++++ manifests/repo.pp | 69 +++ manifests/service.pp | 12 + metadata.json | 108 ++++ spec/acceptance/001_basic_spec.rb | 92 ++++ spec/acceptance/nodesets/centos-7.yml | 16 + spec/acceptance/nodesets/default.yml | 15 + spec/classes/config_spec.rb | 83 +++ spec/classes/coverage_spec.rb | 3 + spec/classes/filebeat_spec.rb | 25 + spec/classes/install/linux_spec.rb | 21 + spec/classes/install/windows_spec.rb | 59 ++ spec/classes/install_spec.rb | 31 ++ spec/classes/params_spec.rb | 11 + spec/classes/repo_spec.rb | 39 ++ spec/classes/service_spec.rb | 24 + spec/default_facts.yml | 9 + spec/defines/prospector_spec.rb | 82 +++ spec/spec_helper.rb | 44 ++ spec/spec_helper_acceptance.rb | 38 ++ spec/unit/facter/filebeat_version_spec.rb | 35 ++ templates/filebeat.yml.erb | 623 ++++++++++++++++++++++ templates/prospector.yml.erb | 198 +++++++ templates/prospector5.yml.erb | 183 +++++++ templates/pure_hash.yml.erb | 2 + 51 files changed, 4169 insertions(+) create mode 100644 .fixtures.yml create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 .pdkignore create mode 100644 .pmtignore create mode 100644 .rspec create mode 100644 .rubocop.yml create mode 100644 .travis.yml create mode 100644 .yardopts create mode 100644 CHANGELOG.md create mode 100644 Gemfile create mode 100644 LICENSE create mode 100644 NOTICE create mode 100644 README.md create mode 100644 Rakefile create mode 100644 appveyor.yml create mode 100644 lib/facter/filebeat_version.rb create mode 100644 manifests/config.pp create mode 100644 manifests/init.pp create mode 100644 manifests/install.pp create mode 100644 manifests/install/freebsd.pp create mode 100644 manifests/install/linux.pp create mode 100644 manifests/install/openbsd.pp create mode 100644 manifests/install/windows.pp create mode 100644 manifests/params.pp create mode 100644 manifests/prospector.pp create mode 100644 manifests/repo.pp create mode 100644 manifests/service.pp create mode 100644 metadata.json create mode 100644 spec/acceptance/001_basic_spec.rb create mode 100644 spec/acceptance/nodesets/centos-7.yml create mode 100644 spec/acceptance/nodesets/default.yml create mode 100644 spec/classes/config_spec.rb create mode 100644 spec/classes/coverage_spec.rb create mode 100644 spec/classes/filebeat_spec.rb create mode 100644 spec/classes/install/linux_spec.rb create mode 100644 spec/classes/install/windows_spec.rb create mode 100644 spec/classes/install_spec.rb create mode 100644 spec/classes/params_spec.rb create mode 100644 spec/classes/repo_spec.rb create mode 100644 spec/classes/service_spec.rb create mode 100644 spec/default_facts.yml create mode 100644 spec/defines/prospector_spec.rb create mode 100644 spec/spec_helper.rb create mode 100644 spec/spec_helper_acceptance.rb create mode 100644 spec/unit/facter/filebeat_version_spec.rb create mode 100644 templates/filebeat.yml.erb create mode 100644 templates/prospector.yml.erb create mode 100644 templates/prospector5.yml.erb create mode 100644 templates/pure_hash.yml.erb diff --git a/.fixtures.yml b/.fixtures.yml new file mode 100644 index 00000000..bef5539d --- /dev/null +++ b/.fixtures.yml @@ -0,0 +1,13 @@ +fixtures: + repositories: + yumrepo_core: + repo: "https://github.com/puppetlabs/puppetlabs-yumrepo_core.git" + puppet_version: ">= 6.0.0" + stdlib: "https://github.com/puppetlabs/puppetlabs-stdlib.git" + apt: + repo: "https://github.com/puppetlabs/puppetlabs-apt.git" + ref: '6.0.0' + powershell: "https://github.com/puppetlabs/puppetlabs-powershell.git" + archive: "https://github.com/voxpupuli/puppet-archive.git" + symlinks: + filebeat: "#{source_dir}" diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..cf0bb69e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.rb eol=lf linguist-language=Puppet +*.erb eol=lf +*.pp eol=lf +*.sh eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..650022e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +.git/ +.*.sw[op] +.metadata +.yardoc +.yardwarns +*.iml +/.bundle/ +/.idea/ +/.vagrant/ +/coverage/ +/bin/ +/doc/ +/Gemfile.local +/Gemfile.lock +/junit/ +/log/ +/pkg/ +/spec/fixtures/manifests/ +/spec/fixtures/modules/ +/tmp/ +/vendor/ +/convert_report.txt +/update_report.txt +.DS_Store diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..e369e3c8 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,41 @@ +--- +stages: + - syntax + - unit + +cache: + paths: + - vendor/bundle + +before_script: + - bundle -v + - rm Gemfile.lock || true + - gem update --system + - gem --version + - bundle -v + - bundle install --without system_tests --path vendor/bundle --jobs $(nproc) + +parallel_spec-Ruby 2.1.9-Puppet ~> 4.0: + stage: unit + image: ruby:2.1.9 + script: + - bundle exec rake parallel_spec + variables: + PUPPET_GEM_VERSION: '~> 4.0' + +syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop-Ruby 2.4.4-Puppet ~> 5.5: + stage: syntax + image: ruby:2.4.4 + script: + - bundle exec rake syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop + variables: + PUPPET_GEM_VERSION: '~> 5.5' + +parallel_spec-Ruby 2.4.4-Puppet ~> 5.5: + stage: unit + image: ruby:2.4.4 + script: + - bundle exec rake parallel_spec + variables: + PUPPET_GEM_VERSION: '~> 5.5' + diff --git a/.pdkignore b/.pdkignore new file mode 100644 index 00000000..650022e5 --- /dev/null +++ b/.pdkignore @@ -0,0 +1,24 @@ +.git/ +.*.sw[op] +.metadata +.yardoc +.yardwarns +*.iml +/.bundle/ +/.idea/ +/.vagrant/ +/coverage/ +/bin/ +/doc/ +/Gemfile.local +/Gemfile.lock +/junit/ +/log/ +/pkg/ +/spec/fixtures/manifests/ +/spec/fixtures/modules/ +/tmp/ +/vendor/ +/convert_report.txt +/update_report.txt +.DS_Store diff --git a/.pmtignore b/.pmtignore new file mode 100644 index 00000000..fb589575 --- /dev/null +++ b/.pmtignore @@ -0,0 +1,20 @@ +docs/ +pkg/ +Gemfile.lock +Gemfile.local +vendor/ +.vendor/ +spec/fixtures/manifests/ +spec/fixtures/modules/ +.vagrant/ +.bundle/ +.ruby-version +coverage/ +log/ +.idea/ +.dependencies/ +.librarian/ +Puppetfile.lock +*.iml +.*.sw? +.yardoc/ diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..16f9cdb0 --- /dev/null +++ b/.rspec @@ -0,0 +1,2 @@ +--color +--format documentation diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..f5a6c2a4 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,122 @@ +--- +require: rubocop-rspec +AllCops: + DisplayCopNames: true + TargetRubyVersion: '2.1' + Include: + - "./**/*.rb" + Exclude: + - bin/* + - ".vendor/**/*" + - "**/Gemfile" + - "**/Rakefile" + - pkg/**/* + - spec/fixtures/**/* + - vendor/**/* + - "**/Puppetfile" + - "**/Vagrantfile" + - "**/Guardfile" +Metrics/LineLength: + Description: People have wide screens, use them. + Max: 200 +GetText/DecorateString: + Description: We don't want to decorate test output. + Exclude: + - spec/* +RSpec/BeforeAfterAll: + Description: Beware of using after(:all) as it may cause state to leak between tests. + A necessary evil in acceptance testing. + Exclude: + - spec/acceptance/**/*.rb +RSpec/HookArgument: + Description: Prefer explicit :each argument, matching existing module's style + EnforcedStyle: each +Style/BlockDelimiters: + Description: Prefer braces for chaining. Mostly an aesthetical choice. Better to + be consistent then. + EnforcedStyle: braces_for_chaining +Style/ClassAndModuleChildren: + Description: Compact style reduces the required amount of indentation. + EnforcedStyle: compact +Style/EmptyElse: + Description: Enforce against empty else clauses, but allow `nil` for clarity. + EnforcedStyle: empty +Style/FormatString: + Description: Following the main puppet project's style, prefer the % format format. + EnforcedStyle: percent +Style/FormatStringToken: + Description: Following the main puppet project's style, prefer the simpler template + tokens over annotated ones. + EnforcedStyle: template +Style/Lambda: + Description: Prefer the keyword for easier discoverability. + EnforcedStyle: literal +Style/RegexpLiteral: + Description: Community preference. See https://github.com/voxpupuli/modulesync_config/issues/168 + EnforcedStyle: percent_r +Style/TernaryParentheses: + Description: Checks for use of parentheses around ternary conditions. Enforce parentheses + on complex expressions for better readability, but seriously consider breaking + it up. + EnforcedStyle: require_parentheses_when_complex +Style/TrailingCommaInArguments: + Description: Prefer always trailing comma on multiline argument lists. This makes + diffs, and re-ordering nicer. + EnforcedStyleForMultiline: comma +Style/TrailingCommaInLiteral: + Description: Prefer always trailing comma on multiline literals. This makes diffs, + and re-ordering nicer. + EnforcedStyleForMultiline: comma +Style/SymbolArray: + Description: Using percent style obscures symbolic intent of array's contents. + EnforcedStyle: brackets +RSpec/MessageSpies: + EnforcedStyle: receive +Style/Documentation: + Exclude: + - lib/puppet/parser/functions/**/* + - spec/**/* +Style/WordArray: + EnforcedStyle: brackets +Style/CollectionMethods: + Enabled: true +Style/MethodCalledOnDoEndBlock: + Enabled: true +Style/StringMethods: + Enabled: true +Layout/EndOfLine: + Enabled: false +Layout/IndentHeredoc: + Enabled: false +Metrics/AbcSize: + Enabled: false +Metrics/BlockLength: + Enabled: false +Metrics/ClassLength: + Enabled: false +Metrics/CyclomaticComplexity: + Enabled: false +Metrics/MethodLength: + Enabled: false +Metrics/ModuleLength: + Enabled: false +Metrics/ParameterLists: + Enabled: false +Metrics/PerceivedComplexity: + Enabled: false +RSpec/DescribeClass: + Enabled: false +RSpec/ExampleLength: + Enabled: false +RSpec/MessageExpectation: + Enabled: false +RSpec/MultipleExpectations: + Enabled: false +RSpec/NestedGroups: + Enabled: false +Style/AsciiComments: + Enabled: false +Style/IfUnlessModifier: + Enabled: false +Style/SymbolProc: + Enabled: false diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..9145857a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,43 @@ +--- +sudo: false +dist: trusty +language: ruby +cache: bundler +before_install: + - bundle -v + - rm -f Gemfile.lock + - gem update --system + - gem --version + - bundle -v +script: + - 'bundle exec rake $CHECK' +bundler_args: --without system_tests +rvm: + - 2.5.0 +env: + global: + - BEAKER_PUPPET_COLLECTION=puppet6 PUPPET_GEM_VERSION="~> 6.0" +matrix: + fast_finish: true + include: + - + env: CHECK="syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop" DEPLOY_TO_FORGE=yes + - + env: CHECK=parallel_spec + - + env: PUPPET_GEM_VERSION="~> 5.0" CHECK=parallel_spec + rvm: 2.4.4 + - + env: PUPPET_GEM_VERSION="~> 4.0" CHECK=parallel_spec + rvm: 2.1.9 +notifications: + email: false +deploy: + provider: puppetforge + user: pcfens + password: + secure: "z1SbP/Hisr5k66XL/ACLsZ/fG7cCpwl8apjZzt/YciWizwReioU2EkLr5tvXdUC10aIH6H7XBdA9XwPqwXa81cIqcdIHlRMIbosMUGYaXcUm1xhctB3GvEDqsxFqdZSHYXax+IR6Wt507Eop+iU3S5pf/zJcp4uSKQVapCMoeVCEQYLRwllgeaqtEUZwqOUwPk31C4YZxwrzmgbIVyXmPrp3SDToXaQm4S4RkayOqHH2lYi8isz3IPPQvDZY5681TBpo35AbsIRbhiLzGlBHbgRaE2dz7J1Gs8MBGFyrtDaPtc9UpbgEmyxgmaPs3NIeZkmfVoosjt2AHRsoMZB7ntaPAQ20mk44ugMhxd5HX8t7QdLPiYQqgA3O4QfKraxPzdEjYVs9Pf7BBgY4JpGSOAD3dlWNK0U40MzKe74cj6dshg9SfIdyf3M3MmI0KIIvdKhpgl8mSIL8MCWjnYYNpQMQDFgyrXvePnkPVlt7zlBxn+LJFFx3VLGNfSWbKavITM/nrvjpFkQZ34mPHTtTUYnT6HVehtwPd5x6ILqYcppEeeiloa4uLWhW/vg0wAOdOBv2IALdAqRMC56ODPK33gFRkX+CclsegtOh2In407njbXXZBQrY5h3SXuEVxZcFhGVTxJIV29viuWFSm7VF0a7IUmEbVrM23bqeaM+aOgs=" + on: + tags: true + all_branches: true + condition: "$DEPLOY_TO_FORGE = yes" diff --git a/.yardopts b/.yardopts new file mode 100644 index 00000000..29c933bc --- /dev/null +++ b/.yardopts @@ -0,0 +1 @@ +--markup markdown diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..0e92c846 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,431 @@ +Changelog +========= + +## Unreleased +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.2...HEAD) + + +## [v3.3.3](https://github.com/pcfens/puppet-filebeat/tree/v3.3.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.2...v3.3.3) + +- Add a harvester limit [\#196](https://github.com/pcfens/puppet-filebeat/pull/196) +- Fix documentaion error [\#198](https://github.com/pcfens/puppet-filebeat/issues/198) +- Fix Puppet 4.10 undefined method error [\#199](https://github.com/pcfens/puppet-filebeat/pull/199) + +## [v3.3.2](https://github.com/pcfens/puppet-filebeat/tree/v3.3.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.1...v3.3.2) + +- Produce valid YAML for prospectors [\#193](https://github.com/pcfens/puppet-filebeat/pull/193) +- Upgrade to PDK 1.7.1 +- Add tests for Puppet 6 +- Add Puppet 6 support to metadata + +## [v3.3.1](https://github.com/pcfens/puppet-filebeat/tree/v3.3.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.3.0...v3.3.1) + +- Add a new `manage_apt` flag for disabling the inclusion of puppetlabs/apt [\#185](https://github.com/pcfens/puppet-filebeat/pull/185) + +## [v3.3.0](https://github.com/pcfens/puppet-filebeat/tree/v3.3.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.2.2...v3.3.0) + +- Allow use of puppet/archive 3.x [\#190](https://github.com/pcfens/puppet-filebeat/pull/190) +- Add support for Docker inputs [\#191](https://github.com/pcfens/puppet-filebeat/pull/191) +- Support puppetlabs/stdlib 5.x + +## [v3.2.2](https://github.com/pcfens/puppet-filebeat/tree/v3.2.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.2.1...v3.2.2) + +- Don't add xpack configuration when it's undef [\#187](https://github.com/pcfens/puppet-filebeat/pull/187) +- Don't disallow using puppetlabs/apt 6.x (check their [changelog](https://forge.puppet.com/puppetlabs/apt/changelog#600-2018-08-24) as this release drops support for Puppet pre 4.7) [\#186](https://github.com/pcfens/puppet-filebeat/pull/186) +- Use the latest PDK + +## [v3.2.1](https://github.com/pcfens/puppet-filebeat/tree/v3.2.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.2.0...v3.2.1) + +- Fetch the suse repository signing key over https [\#176](https://github.com/pcfens/puppet-filebeat/issues/176) + +## [v3.2.0](https://github.com/pcfens/puppet-filebeat/tree/v3.2.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.1.0...v3.2.0) + +- Add support for xpack monitoring [\#172](https://github.com/pcfens/puppet-filebeat/pull/172) +- Add support for OpenBSD [\#173](https://github.com/pcfens/puppet-filebeat/pull/173) +- Set filebeat_version to false when filebeat isn't installed [\#175](https://github.com/pcfens/puppet-filebeat/pull/175) + +## [v3.1.0](https://github.com/pcfens/puppet-filebeat/tree/v3.1.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.0.1...v3.1.0) + +- Manage filebeat modules as an array [\#168](https://github.com/pcfens/puppet-filebeat/pull/168) + +## [v3.0.1](https://github.com/pcfens/puppet-filebeat/tree/v3.0.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v3.0.0...v3.0.1) + +- Fix array validation in prospector defined resource [\#166](https://github.com/pcfens/puppet-filebeat/pull/166) + +## [v3.0.0](https://github.com/pcfens/puppet-filebeat/tree/v3.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.4.0...v3.0.0) + +Potentially Breaking Change: +- Make filebeat 6 the default version. + +Non-breaking changes: +- Allow setup entries in configuration [\#152](https://github.com/pcfens/puppet-filebeat/pull/152), [\#146](https://github.com/pcfens/puppet-filebeat/issues/146) +- Processors should be an array of hashes [\#157](https://github.com/pcfens/puppet-filebeat/pull/157), [\#156](https://github.com/pcfens/puppet-filebeat/issues/156) +- Validate URLs using stdlib [\#158](https://github.com/pcfens/puppet-filebeat/pull/158) +- Use external configuration setup for Filebeat 6+ [\#153](https://github.com/pcfens/puppet-filebeat/issues/153) +- Use version subcommand when determining version [\#159](https://github.com/pcfens/puppet-filebeat/issues/159) +- Add processors support to prospectors [\#162](https://github.com/pcfens/puppet-filebeat/pull/162) +- Update unsupported OS Family notice [\#161](https://github.com/pcfens/puppet-filebeat/pull/161) +- Use Puppet 4+ data types for prospectors [\#165](https://github.com/pcfens/puppet-filebeat/pull/165) +- Fix windows validation command [\#164](https://github.com/pcfens/puppet-filebeat/pull/164), [\#163](https://github.com/pcfens/puppet-filebeat/issues/163) + +## [v2.4.0](https://github.com/pcfens/puppet-filebeat/tree/v2.4.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.3.0...v2.4.0) + +- Add support for FreeBSD [\#130](https://github.com/pcfens/puppet-filebeat/pull/130) +- Add support for Archlinux [\#147](https://github.com/pcfens/puppet-filebeat/pull/147) + +## [v2.3.0](https://github.com/pcfens/puppet-filebeat/tree/v2.3.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.2.0...v2.3.0) + +- Add support for Filebeat 6 [\#141](https://github.com/pcfens/puppet-filebeat/pull/141) +- Add Support for hash.random in Kafka output [\#142](https://github.com/pcfens/puppet-filebeat/pull/142) + +## [v2.2.0](https://github.com/pcfens/puppet-filebeat/tree/v2.2.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.1.0...v2.2.0) + +- Support pipeline configurations in prospectors [\#134](https://github.com/pcfens/puppet-filebeat/pull/134) +- Fix regex for validating download URL [\#135](https://github.com/pcfens/puppet-filebeat/pull/135) +- Overhaul testing + +## [v2.1.0](https://github.com/pcfens/puppet-filebeat/tree/v2.1.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v2.0.0...v2.1.0) + +- Change beat_name configuration parameter to name [\#126](https://github.com/pcfens/puppet-filebeat/issues/126) +- Make configuration directory/file ownership configurable [\#127](https://github.com/pcfens/puppet-filebeat/issues/127) + +## [v2.0.0](https://github.com/pcfens/puppet-filebeat/tree/v2.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v1.0.0...v2.0.0) + +- Drop support for Puppet 3 +- Drop support for Filebeat versions before 5 +- Add support for Puppet 5 +- Use a generic template by default +- Remove processor defined type (create it in the config template) +- Add a flag to disable validating the configuration (`disable_config_test`) + +## [v1.0.0](https://github.com/pcfens/puppet-filebeat/tree/v1.0.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.12.1...v1.0.0) + +- This is the last release with support for Filebeat versions prior to 5 +- Last release with support for Puppet 3 +- Add Logstash SSL support [\#121](https://github.com/pcfens/puppet-filebeat/pull/121) +- Add ES loadbalance support [\#119](https://github.com/pcfens/puppet-filebeat/pull/119) + +The next major release will be a breaking release for anyone using processors. + +## [v0.12.1](https://github.com/pcfens/puppet-filebeat/tree/v0.12.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.12.0...v0.12.1) + +- Add support for SSL in Logstash [\#117](https://github.com/pcfens/puppet-filebeat/pull/117) + +## [v0.12.0](https://github.com/pcfens/puppet-filebeat/tree/v0.12.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.11.2...v0.12.0) + +Windows users: you may see a restart and replacement of your existing filebeat directory. +There's a very brief discussion of the change in the [pull request](https://github.com/pcfens/puppet-filebeat/pull/113#issuecomment-307628477) + +- Support upgrades in Windows [\#113](https://github.com/pcfens/puppet-filebeat/pull/113) +- Add optional repo_priority parameter [\#110](https://github.com/pcfens/puppet-filebeat/pull/110) +- Update external dependencies, including pinning apt version + +## [v0.11.2](https://github.com/pcfens/puppet-filebeat/tree/v0.11.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.11.1...v0.11.2) + +- Explicitly support newer versions of the powershell modules [\#105](https://github.com/pcfens/puppet-filebeat/issues/105) +- Support kafka codec.format module [\#106](https://github.com/pcfens/puppet-filebeat/pull/106) +- The `add_locale` processor doesnt' require parameters [\#104](https://github.com/pcfens/puppet-filebeat/pull/104) + +## [v0.11.1](https://github.com/pcfens/puppet-filebeat/tree/v0.11.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.11.0...v0.11.1) + +- Restore puppet 3.x compatibility regression ([PUP-2523](https://tickets.puppetlabs.com/browse/PUP-2523)) + +## [v0.11.0](https://github.com/pcfens/puppet-filebeat/tree/v0.11.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.4...v0.11.0) + +**Breaking Changes** +- Processors are managed by their own defined resource (slight syntax change) [\#85](https://github.com/pcfens/puppet-filebeat/pull/85) +- The registry file has likely moved because of an upstream change. Moving this file + can cause problems (duplicate or missed log entries), so you may want to point it + to your existing file (check in /.filebeat on Linux systems) + +**Normal Enhancements/Bugfixes** +- Support proxy for windows file downloads [\#90](https://github.com/pcfens/puppet-filebeat/pull/90) +- Setting `package_ensure` to absent removes puppet managed files and removes the package +- Add support for index conditional output to elasticsearch [\#97](https://github.com/pcfens/puppet-filebeat/pull/97) +- Add support for a conditional pipeline for elasticsearch [\#98](https://github.com/pcfens/puppet-filebeat/pull/98) +- Template should check for nil instead of undef [\#63](https://github.com/pcfens/puppet-filebeat/issues/63) +- Support for the round_robin and group_events parameters in kafka outputs [\#100](https://github.com/pcfens/puppet-filebeat/pull/100) + +## [v0.10.4](https://github.com/pcfens/puppet-filebeat/tree/v0.10.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.3...v0.10.4) + +- Add output.console support to the config template [\#91](https://github.com/pcfens/puppet-filebeat/issues/91) +- Support puppet with strict variables enabled [\#92](https://github.com/pcfens/puppet-filebeat/issues/92) + +## [v0.10.3](https://github.com/pcfens/puppet-filebeat/tree/v0.10.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.2...v0.10.3) + +- Allow non-SSL downloads of windows filebeat zipfile [\#82](https://github.com/pcfens/puppet-filebeat/pull/82) +- Basic support of processors in puppet <4.x [\#79](https://github.com/pcfens/puppet-filebeat/issues/79) (See note above) +- Confine the filebeat_version fact in a way that works in Ruby 1.8.7 [\#88](https://github.com/pcfens/puppet-filebeat/pull/88) + +## [v0.10.2](https://github.com/pcfens/puppet-filebeat/tree/v0.10.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.1...v0.10.2) + +- Add close_older and force_close_files within prospector v5 [\#77](https://github.com/pcfens/puppet-filebeat/pull/77) + +## [v0.10.1](https://github.com/pcfens/puppet-filebeat/tree/v0.10.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.10.0...v0.10.1) + +- Support harvesting symlinks [\#74](https://github.com/pcfens/puppet-filebeat/pull/74) +- Fix windows config file validation command [\#75](https://github.com/pcfens/puppet-filebeat/issues/75) + +## [v0.10.0](https://github.com/pcfens/puppet-filebeat/tree/v0.10.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.2...v0.10.0) + +- Add support for JSON decoding [\#72](https://github.com/pcfens/puppet-filebeat/pull/72) + +## [v0.9.2](https://github.com/pcfens/puppet-filebeat/tree/v0.9.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.1...v0.9.2) + +- Add support for close_* and clean_* parameters in prospectors [\#70](https://github.com/pcfens/puppet-filebeat/pull/70) + +## [v0.9.1](https://github.com/pcfens/puppet-filebeat/tree/v0.9.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.9.0...v0.9.1) + +- Fix yaml syntax around filebeat processors [\#71](https://github.com/pcfens/puppet-filebeat/pull/71) + +## [v0.9.0](https://github.com/pcfens/puppet-filebeat/tree/v0.9.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.7...v0.9.0) + +- Add support for tags in prospectors [\#68](https://github.com/pcfens/puppet-filebeat/pull/68) +- Add support for filebeat processors [\#69](https://github.com/pcfens/puppet-filebeat/pull/69) +- Fix the `filebeat_version` fact in Windows [\#59](https://github.com/pcfens/puppet-filebeat/issues/59) +- Validate configuration files before notifying the filebeat service +- Update the Windows install URL to the latest version + +## [v0.8.7](https://github.com/pcfens/puppet-filebeat/tree/v0.8.7) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.6...v0.8.7) + +- Update windows URL to the latest 5.x release +- Remove nil values before rendering the template [\#65](https://github.com/pcfens/puppet-filebeat/pull/65) + +## [v0.8.6](https://github.com/pcfens/puppet-filebeat/tree/v0.8.6) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.5...v0.8.6) + +- Sort field keys [\#55](https://github.com/pcfens/puppet-filebeat/pull/55), +[\#57](https://github.com/pcfens/puppet-filebeat/issues/57) +- Refresh the filebeat service when packages are updated [\#56](https://github.com/pcfens/puppet-filebeat/issues/56) + + +## [v0.8.5](https://github.com/pcfens/puppet-filebeat/tree/v0.8.5) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.4...v0.8.5) + +- Check the kafka partition hash before checking for sub-hashes [\#54](https://github.com/pcfens/puppet-filebeat/pull/54) + +## [v0.8.4](https://github.com/pcfens/puppet-filebeat/tree/v0.8.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.3...v0.8.4) + +- Fix regression: Add the SSL label to the filebeat 5 template. [\#53](https://github.com/pcfens/puppet-filebeat/pull/53) + +## [v0.8.3](https://github.com/pcfens/puppet-filebeat/tree/v0.8.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.2...v0.8.3) + +- Don't use a possibly undefined array's length to determine if it should be + iterated over [\#52](https://github.com/pcfens/puppet-filebeat/pull/52) + +## [v0.8.2](https://github.com/pcfens/puppet-filebeat/tree/v0.8.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.1...v0.8.2) + +- Correctly set document type for v5 prospectors [\#51](https://github.com/pcfens/puppet-filebeat/pull/51) + +## [v0.8.1](https://github.com/pcfens/puppet-filebeat/tree/v0.8.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.8.0...v0.8.1) + +- Don't manage the apt-transport-https package on Debian systems [\#49](https://github.com/pcfens/puppet-filebeat/pull/49) +- undefined values shouldn't be rendered by the filebeat5 template [\#50](https://github.com/pcfens/puppet-filebeat/pull/50) + +## [v0.8.0](https://github.com/pcfens/puppet-filebeat/tree/v0.8.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.7.4...v0.8.0) + +**Enhancements** +- Add support for Filebeat v5. + +If you use this module on a system with filebeat 1.x installed, and you keep your current parameters +nothing will change. Setting `major_version` to '5' will modify the configuration template and update +package repositories, but won't update the package itself. To update the package set the +`package_ensure` parameter to at least 5.0.0. + +- Add a parameter `use_generic_template` that uses a more generic version of the configuration + template. The generic template is more future proof (if types are correct), but looks + very different than the example file. + + +## [v0.7.4](https://github.com/pcfens/puppet-filebeat/tree/v0.7.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.7.2...v0.7.4) + +Version 0.7.3 was never released even though it is tagged. + +- Fixed some testing issues that were caused by changes to external resources + +**Fixed Bugs** +- Some redis configuration options were not generated as integers [\#38](https://github.com/pcfens/puppet-filebeat/issues/38) + +## [v0.7.2](https://github.com/pcfens/puppet-filebeat/tree/v0.7.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.7.1...v0.7.2) + +- Wrap regular expressions in single quotes [\#31](https://github.com/pcfens/puppet-filebeat/pull/31) and [\#35](https://github.com/pcfens/puppet-filebeat/pull/35) +- Use the default Windows temporary folder (C:\Windows\Temp) by default [\#33](https://github.com/pcfens/puppet-filebeat/pull/33) + +## [v0.7.1](https://github.com/pcfens/puppet-filebeat/tree/v0.7.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.7.0...v0.7.1) + +- Allow the config file to be written to an alternate location. Be sure and read limitations before you use this. + +**Fixed Bugs** +- Add elasticsearch and logstash port setting to Ruby 1.8 template + [\#29](https://github.com/pcfens/puppet-filebeat/issues/29) + +## [v0.7.0](https://github.com/pcfens/puppet-filebeat/tree/v0.7.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.3...v0.7.0) + +- Setting the `prospectors_merge` parameter to true will create prospectors across multiple hiera levels + using `hiera_hash()` [\#25](https://github.com/pcfens/puppet-filebeat/pull/25) +- No longer manage the windows temp directory where the Filebeat download is kept. The assumption is made + that the directory exists and is writable by puppet. +- Update the default windows download to Filebeat version 1.2.3 +- Add redis output to the Ruby 1.8 template +- Wrap include_lines and exclude_lines array elements in quotes [\#28](https://github.com/pcfens/puppet-filebeat/issues/28) + +**Fixed Bugs** +- SLES repository and metaparameters didn't match [\#25](https://github.com/pcfens/puppet-filebeat/pull/25) + +## [v0.6.3](https://github.com/pcfens/puppet-filebeat/tree/v0.6.3) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.2...v0.6.3) + +**Fixed Bugs** +- Spool size default should match upstream [\#24](https://github.com/pcfens/puppet-filebeat/pull/24) +- Repository names now match notification parameters Part of [\#25](https://github.com/pcfens/puppet-filebeat/pull/25) + +## [v0.6.2](https://github.com/pcfens/puppet-filebeat/tree/v0.6.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.1...v0.6.2) + +**Fixed Bugs** +- Fix the other certificate_key typo in Ruby 1.8 template +[\#23](https://github.com/pcfens/puppet-filebeat/issues/23) + + +## [v0.6.1](https://github.com/pcfens/puppet-filebeat/tree/v0.6.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.6.0...v0.6.1) + +**Fixed Bugs** +- Fix typo in Ruby 1.8 template [\#23](https://github.com/pcfens/puppet-filebeat/issues/23) + + +## [v0.6.0](https://github.com/pcfens/puppet-filebeat/tree/v0.6.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.8...v0.6.0) + +- Add the `close_older` parameter to support the option of the same name in filebeat 1.2.0 +- Add support for the `publish_async` parameter. + +**Fixed Bugs** +- Added limited, but improved support for Ruby versions pre-1.9.1 by fixing the hash sort issue +[\#20](https://github.com/pcfens/puppet-filebeat/issues/20) + +## [v0.5.8](https://github.com/pcfens/puppet-filebeat/tree/v0.5.8) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.7...v0.5.8) + +**Fixed Bugs** +- `doc_type` is now used in the documentation instead of the deprecated `log_type` + [\#17](https://github.com/pcfens/puppet-filebeat/pull/17) +- RedHat based systems should be using the redhat service provider. + [\#18](https://github.com/pcfens/puppet-filebeat/pull/18) + + +## [v0.5.7](https://github.com/pcfens/puppet-filebeat/tree/v0.5.7) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.6...v0.5.7) + +**Fixed Bugs** +- Some configuration parameters should be rendered as integers, not strings + [\#15](https://github.com/pcfens/puppet-filebeat/pull/15) + + +## [v0.5.6](https://github.com/pcfens/puppet-filebeat/tree/v0.5.6) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.5...v0.5.6) + +**Fixed Bugs** +- Configuration files should use the `conf_template` parameter [\#14](https://github.com/pcfens/puppet-filebeat/pull/14) + +## [v0.5.5](https://github.com/pcfens/puppet-filebeat/tree/v0.5.5) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.4...v0.5.5) + +**Fixed Bugs** +- `rotate_every_kb` and `number_of_files` parameters in file outputs should be + explicitly integers to keep filebeat happy. [\#13](https://github.com/pcfens/puppet-filebeat/issues/13) + +## [v0.5.4](https://github.com/pcfens/puppet-filebeat/tree/v0.5.4) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.2...v0.5.4) + +**Fixed Bugs** +- Fix template regression in v0.5.3 + +## [v0.5.2](https://github.com/pcfens/puppet-filebeat/tree/v0.5.2) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.1...v0.5.2) + +- Use the anchor pattern instead of contain so that older versions of puppet + are supported [\#12](https://github.com/pcfens/puppet-filebeat/pull/12) + +## [v0.5.1](https://github.com/pcfens/puppet-filebeat/tree/v0.5.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.5.0...v0.5.1) + +- Update metadata to reflect which versions of puppet are supported. + +## [v0.5.0](https://github.com/pcfens/puppet-filebeat/tree/v0.5.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.4.1...v0.5.0) + +- For prospectors, deprecate `log_type` in favor of `doc_type` to better + match the actual configuration parameter. `document_type` is not used because + it causes errors when running with a puppet master. `log_type` will be fully + removed before module version 1.0. + [\#9](https://github.com/pcfens/puppet-filebeat/issues/9) + +**New Features** +- Add support for `exclude_files`, `exclude_lines`, `include_lines`, and `multiline`. + Use of the new parameters requires a filebeat version >= 1.1 + ([\#10](https://github.com/pcfens/puppet-filebeat/issues/10), [\#11](https://github.com/pcfens/puppet-filebeat/issues/11)) + +## [v0.4.1](https://github.com/pcfens/puppet-filebeat/tree/v0.4.1) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.4.0...v0.4.1) + +**Fixed Bugs** +- Fix links in documentation to match the updated documentation + +**New Features** +- Change repository resource names to beats (e.g. apt::source['beats'], etc.), + and only declare them if they haven't already been declared. This way we only + have one module for all beats modules managed through puppet. + +## [v0.4.0](https://github.com/pcfens/puppet-filebeat/tree/v0.4.0) +[Full Changelog](https://github.com/pcfens/puppet-filebeat/compare/v0.3.1...v0.4.0) + +This is the first release that includes changelog. Since v0.3.1: + +**Fixed Bugs** +- 'fields' parse error in prospector.yml template [\#7](https://github.com/pcfens/puppet-filebeat/pull/7) + +**New Features** +- Windows support [\#3](https://github.com/pcfens/puppet-filebeat/pull/3) + - Requires the [`puppetlabs/powershell`](https://forge.puppetlabs.com/puppetlabs/powershell) + and [`lwf/remote_file`](https://forge.puppetlabs.com/lwf/remote_file) modules. +- Config file and folder permissions can be managed [\#8](https://github.com/pcfens/puppet-filebeat/pull/8) diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..626d661b --- /dev/null +++ b/Gemfile @@ -0,0 +1,70 @@ +source ENV['GEM_SOURCE'] || 'https://rubygems.org' + +def location_for(place_or_version, fake_version = nil) + git_url_regex = %r{\A(?(https?|git)[:@][^#]*)(#(?.*))?} + file_url_regex = %r{\Afile:\/\/(?.*)} + + if place_or_version && (git_url = place_or_version.match(git_url_regex)) + [fake_version, { git: git_url[:url], branch: git_url[:branch], require: false }].compact + elsif place_or_version && (file_url = place_or_version.match(file_url_regex)) + ['>= 0', { path: File.expand_path(file_url[:path]), require: false }] + else + [place_or_version, { require: false }] + end +end + +ruby_version_segments = Gem::Version.new(RUBY_VERSION.dup).segments +minor_version = ruby_version_segments[0..1].join('.') + +group :development do + gem "fast_gettext", '1.1.0', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.1.0') + gem "fast_gettext", require: false if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new('2.1.0') + gem "json_pure", '<= 2.0.1', require: false if Gem::Version.new(RUBY_VERSION.dup) < Gem::Version.new('2.0.0') + gem "json", '= 1.8.1', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9') + gem "json", '<= 2.0.4', require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.4.4') + gem "puppet-module-posix-default-r#{minor_version}", require: false, platforms: [:ruby] + gem "puppet-module-posix-dev-r#{minor_version}", require: false, platforms: [:ruby] + gem "puppet-module-win-default-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] + gem "puppet-module-win-dev-r#{minor_version}", require: false, platforms: [:mswin, :mingw, :x64_mingw] +end + +puppet_version = ENV['PUPPET_GEM_VERSION'] +facter_version = ENV['FACTER_GEM_VERSION'] +hiera_version = ENV['HIERA_GEM_VERSION'] + +gems = {} + +gems['puppet'] = location_for(puppet_version) + +# If facter or hiera versions have been specified via the environment +# variables + +gems['facter'] = location_for(facter_version) if facter_version +gems['hiera'] = location_for(hiera_version) if hiera_version + +if Gem.win_platform? && puppet_version =~ %r{^(file:///|git://)} + # If we're using a Puppet gem on Windows which handles its own win32-xxx gem + # dependencies (>= 3.5.0), set the maximum versions (see PUP-6445). + gems['win32-dir'] = ['<= 0.4.9', require: false] + gems['win32-eventlog'] = ['<= 0.6.5', require: false] + gems['win32-process'] = ['<= 0.7.5', require: false] + gems['win32-security'] = ['<= 0.2.5', require: false] + gems['win32-service'] = ['0.8.8', require: false] +end + +gems.each do |gem_name, gem_params| + gem gem_name, *gem_params +end + +# Evaluate Gemfile.local and ~/.gemfile if they exist +extra_gemfiles = [ + "#{__FILE__}.local", + File.join(Dir.home, '.gemfile'), +] + +extra_gemfiles.each do |gemfile| + if File.file?(gemfile) && File.readable?(gemfile) + eval(File.read(gemfile), binding) + end +end +# vim: syntax=ruby diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..a945b979 --- /dev/null +++ b/NOTICE @@ -0,0 +1,13 @@ +Copyright 2016 College of William & Mary + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..79d9dab9 --- /dev/null +++ b/README.md @@ -0,0 +1,395 @@ +# puppet-filebeat + +[![Build Status](https://travis-ci.org/pcfens/puppet-filebeat.svg?branch=master)](https://travis-ci.org/pcfens/puppet-filebeat) + +#### Table of Contents + +1. [Description](#description) +2. [Setup - The basics of getting started with filebeat](#setup) + - [What filebeat affects](#what-filebeat-affects) + - [Setup requirements](#setup-requirements) + - [Beginning with filebeat](#beginning-with-filebeat) +3. [Usage - Configuration options and additional functionality](#usage) + - [Adding a prospector](#adding-a-prospector) + - [Multiline Logs](#multiline-logs) + - [JSON logs](#json-logs) + - [Prospectors in hiera](#prospectors-in-hiera) + - [Usage on Windows](#usage-on-windows) + - [Processors](#processors) +4. [Reference](#reference) + - [Public Classes](#public-classes) + - [Private Classes](#private-classes) + - [Public Defines](#public-defines) +5. [Limitations - OS compatibility, etc.](#limitations) + - [Pre-1.9.1 Ruby](#pre-191-ruby) + - [Using config_file](#using-config_file) +6. [Development - Guide for contributing to the module](#development) + +## Description + +The `filebeat` module installs and configures the [filebeat log shipper](https://www.elastic.co/products/beats/filebeat) maintained by elastic. + +## Setup + +### What filebeat affects + +By default `filebeat` adds a software repository to your system, and installs filebeat along +with required configurations. + +### Upgrading to Filebeat 6.x + +To upgrade to Filebeat 6.x, simply set `$filebeat::major_version` to `6` and `$filebeat::package_ensure` to `latest` (or whichever version of 6.x you want, just not present). + + +### Setup Requirements + +The `filebeat` module depends on [`puppetlabs/stdlib`](https://forge.puppetlabs.com/puppetlabs/stdlib), and on +[`puppetlabs/apt`](https://forge.puppetlabs.com/puppetlabs/apt) on Debian based systems. + +### Beginning with filebeat + +`filebeat` can be installed with `puppet module install pcfens-filebeat` (or with r10k, librarian-puppet, etc.) + +The only required parameter, other than which files to ship, is the `outputs` parameter. + +## Usage + +All of the default values in filebeat follow the upstream defaults (at the time of writing). + +To ship files to [elasticsearch](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#elasticsearch-output): +```puppet +class { 'filebeat': + outputs => { + 'elasticsearch' => { + 'hosts' => [ + 'http://localhost:9200', + 'http://anotherserver:9200' + ], + 'loadbalance' => true, + 'cas' => [ + '/etc/pki/root/ca.pem', + ], + }, + }, +} + +``` + +To ship log files through [logstash](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#logstash-output): +```puppet +class { 'filebeat': + outputs => { + 'logstash' => { + 'hosts' => [ + 'localhost:5044', + 'anotherserver:5044' + ], + 'loadbalance' => true, + }, + }, +} + +``` + +[Shipper](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-shipper) and +[logging](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html#configuration-logging) options +can be configured the same way, and are documented on the [elastic website](https://www.elastic.co/guide/en/beats/filebeat/current/index.html). + +### Adding a prospector + +Prospectors are processes that ship log files to elasticsearch or logstash. They can +be defined as a hash added to the class declaration (also used for automatically creating +prospectors using hiera), or as their own defined resources. + +At a minimum, the `paths` parameter must be set to an array of files or blobs that should +be shipped. `doc_type` is what logstash views as the type parameter if you'd like to +apply conditional filters. + +```puppet +filebeat::prospector { 'syslogs': + paths => [ + '/var/log/auth.log', + '/var/log/syslog', + ], + doc_type => 'syslog-beat', +} +``` + +#### Multiline Logs + +Filebeat prospectors can handle multiline log entries. The `multiline` +parameter accepts a hash containing `pattern`, `negate`, `match`, `max_lines`, and `timeout` +as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html). + +#### JSON Logs + +Filebeat prospectors (versions >= 5.0) can natively decode JSON objects if they are stored one per line. The `json` +parameter accepts a hash containing `message_key`, `keys_under_root`, `overwrite_keys`, and `add_error_key` +as documented in the filebeat [configuration documentation](https://www.elastic.co/guide/en/beats/filebeat/5.5/configuration-filebeat-options.html#config-json). + +### Prospectors in Hiera + +Prospectors can be defined in hiera using the `prospectors` parameter. By default, hiera will not merge +prospector declarations down the hiera hierarchy. That behavior can be changed by configuring the +[lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) +flag. + +### Usage on Windows + +When installing on Windows, this module will download the windows version of Filebeat from +[elastic](https://www.elastic.co/downloads/beats/filebeat) to `C:\Temp` by default. The directory +can be overridden using the `tmp_dir` parameter. `tmp_dir` is not managed by this module, +but is expected to exist as a directory that puppet can write to. + +### Processors + +Filebeat 5.0 and greater includes a new libbeat feature for filtering and/or enhancing all +exported data through processors before being sent to the configured output(s). They can be +defined as a hash added to the class declaration (also used for automatically creating +processors using hiera), or as their own defined resources. + +To drop the offset and input_type fields from all events: + +```puppet +class{"filebeat": + processors => { + "drop_fields" => { + "fields" => ["input_type", "offset"], + }, + }, +} +``` + +To drop all events that have the http response code equal to 200: + +```puppet +class{"filebeat": + processors => { + "drop_event" => { + "when" => {"equals" => {"http.code" => 200}} + }, + }, +} +``` + +Now to combine these examples into a single definition: + +```puppet +class{"filebeat": + processors => { + "drop_fields" => { + "params" => {"fields" => ["input_type", "offset"]}, + "priority" => 1, + }, + "drop_event" => { + "when" => {"equals" => {"http.code" => 200}}, + "priority: => 2, + }, + }, +} +``` + +For more information please review the documentation [here](https://www.elastic.co/guide/en/beats/filebeat/5.1/configuration-processors.html). + +#### Processors in Hiera + +Processors can be declared in hiera using the `processors` parameter. By default, hiera will not merge +processor declarations down the hiera hierarchy. That behavior can be changed by configuring the +[lookup_options](https://docs.puppet.com/puppet/latest/reference/lookup_quick.html#setting-lookupoptions-in-data) +flag. + +## Reference + - [**Public Classes**](#public-classes) + - [Class: filebeat](#class-filebeat) + - [**Private Classes**](#private-classes) + - [Class: filebeat::config](#class-filebeatconfig) + - [Class: filebeat::install](#class-filebeatinstall) + - [Class: filebeat::params](#class-filebeatparams) + - [Class: filebeat::repo](#class-filebeatrepo) + - [Class: filebeat::service](#class-filebeatservice) + - [Class: filebeat::install::linux](#class-filebeatinstalllinux) + - [Class: filebeat::install::windows](#class-filebeatinstallwindows) + - [**Public Defines**](#public-defines) + - [Define: filebeat::prospector](#define-filebeatprospector) + - [Define: filebeat::processors](#define-filebeatprocessor) + +### Public Classes + +#### Class: `filebeat` + +Installs and configures filebeat. + +**Parameters within `filebeat`** +- `package_ensure`: [String] The ensure parameter for the filebeat package If set to absent, + prospectors and processors passed as parameters are ignored and everything managed by + puppet will be removed. (default: present) +- `manage_repo`: [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) +- `major_version`: [Enum] The major version of Filebeat to install. Should be either `'5'` or `'6'`. The default value is `'6'`, except + for OpenBSD 6.3 and earlier, which has a default value of `'5'`. +- `service_ensure`: [String] The ensure parameter on the filebeat service (default: running) +- `service_enable`: [String] The enable parameter on the filebeat service (default: true) +- `param repo_priority`: [Integer] Repository priority. yum and apt supported (default: undef) +- `service_provider`: [String] The provider parameter on the filebeat service (default: on RedHat based systems use redhat, otherwise undefined) +- `spool_size`: [Integer] How large the spool should grow before being flushed to the network (default: 2048) +- `idle_timeout`: [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) +- `publish_async`: [Boolean] If set to true filebeat will publish while preparing the next batch of lines to transmit (default: false) +- `registry_file`: [String] The registry file used to store positions, must be an absolute path (default is OS dependent - see params.pp) +- `config_file`: [String] Where the configuration file managed by this module should be placed. If you think + you might want to use this, read the [limitations](#using-config_file) first. Defaults to the location + that filebeat expects for your operating system. +- `config_dir`: [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) +- `config_dir_mode`: [String] The permissions mode set on the configuration directory (default: 0755) +- `config_dir_owner`: [String] The owner of the configuration directory (default: root). Linux only. +- `config_dir_group`: [String] The group of the configuration directory (default: root). Linux only. +- `config_file_mode`: [String] The permissions mode set on configuration files (default: 0644) +- `config_file_owner`: [String] The owner of the configuration files, including prospectors (default: root). Linux only. +- `config_file_group`: [String] The group of the configuration files, including prospectors (default: root). Linux only. +- `purge_conf_dir`: [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged +- `outputs`: [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) +- `shipper`: [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) +- `logging`: [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) +- `modules`: [Array] Will be converted to YAML to create the optional modules section of the filebeat config (see documentation) +- `conf_template`: [String] The configuration template to use to generate the main filebeat.yml config file. +- `download_url`: [String] The URL of the zip file that should be downloaded to install filebeat (windows only) +- `install_dir`: [String] Where filebeat should be installed (windows only) +- `tmp_dir`: [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only) +- `shutdown_timeout`: [String] How long filebeat waits on shutdown for the publisher to finish sending events +- `beat_name`: [String] The name of the beat shipper (default: hostname) +- `tags`: [Array] A list of tags that will be included with each published transaction +- `queue_size`: [String] The internal queue size for events in the pipeline +- `max_procs`: [Number] The maximum number of CPUs that can be simultaneously used +- `fields`: [Hash] Optional fields that should be added to each event output +- `fields_under_root`: [Boolean] If set to true, custom fields are stored in the top level instead of under fields +- `disable_config_test`: [Boolean] If set to true, configuration tests won't be run on config files before writing them. +- `processors`: [Hash] Processors that should be configured. +- `prospectors`: [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera +- `setup`: [Hash] Setup that will be created. Commonly used to create setup using hiera +- `xpack`: [Hash] XPack configuration to pass to filebeat + +### Private Classes + +#### Class: `filebeat::config` + +Creates the configuration files required for filebeat (but not the prospectors) + +#### Class: `filebeat::install` + +Calls the correct installer class based on the kernel fact. + +#### Class: `filebeat::params` + +Sets default parameters for `filebeat` based on the OS and other facts. + +#### Class: `filebeat::repo` + +Installs the yum or apt repository for the system package manager to install filebeat. + +#### Class: `filebeat::service` + +Configures and manages the filebeat service. + +#### Class: `filebeat::install::linux` + +Install the filebeat package on Linux kernels. + +#### Class: `filebeat::install::windows` + +Downloads, extracts, and installs the filebeat zip file in Windows. + +### Public Defines + +#### Define: `filebeat::prospector` + +Installs a configuration file for a prospector. + +Be sure to read the [filebeat configuration details](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html) +to fully understand what these parameters do. + +**Parameters for `filebeat::prospector`** + - `ensure`: The ensure parameter on the prospector configuration file. (default: present) + - `paths`: [Array] The paths, or blobs that should be handled by the prospector. (required only if input_type is not _docker_) + - `containers_ids`: [Array] If input_type is _docker_, the list of Docker container ids to read the logs from. (default: '*') + - `containers_path`: [String] If input_type is _docker_, the path from where the logs should be read from. (default: /var/log/docker/containers) + - `containers_stream`: [String] If input_type is _docker_, read from the specified stream only. (default: all) + - `combine_partial`: [Boolean] If input_type is _docker_, enable partial messages joining. (default: false) + - `cri_parse_flags`: [Boolean] If input_type is _docker_, enable CRI flags parsing from the log file. (default: false) + - `exclude_files`: [Array] Files that match any regex in the list are excluded from filebeat (default: []) + - `encoding`: [String] The file encoding. (default: plain) + - `input_type`: [String] log, docker or stdin - where filebeat reads the log from (default:log) + - `fields`: [Hash] Optional fields to add information to the output (default: {}) + - `fields_under_root`: [Boolean] Should the `fields` parameter fields be stored at the top level of indexed documents. + - `ignore_older`: [String] Files older than this field will be ignored by filebeat (default: ignore nothing) + - `close_older`: [String] Files that haven't been modified since `close_older`, they'll be closed. New + modifications will be read when files are scanned again according to `scan_frequency`. (default: 1h) + - `log_type`: [String] \(Deprecated - use `doc_type`\) The document_type setting (optional - default: log) + - `doc_type`: [String] The event type to used for published lines, used as type field in logstash + and elasticsearch (optional - default: log) + - `scan_frequency`: [String] How often should the prospector check for new files (default: 10s) + - `harvester_buffer_size`: [Integer] The buffer size the harvester uses when fetching the file (default: 16384) + - `tail_files`: [Boolean] If true, filebeat starts reading new files at the end instead of the beginning (default: false) + - `backoff`: [String] How long filebeat should wait between scanning a file after reaching EOF (default: 1s) + - `max_backoff`: [String] The maximum wait time to scan a file for new lines to ship (default: 10s) + - `backoff_factor`: [Integer] `backoff` is multiplied by this parameter until `max_backoff` is reached to + determine the actual backoff (default: 2) + - `force_close_files`: [Boolean] Should filebeat forcibly close a file when renamed (default: false) + - `pipeline`: [String] Filebeat can be configured for a different ingest pipeline for each prospector (default: undef) + - `include_lines`: [Array] A list of regular expressions to match the lines that you want to include. + Ignored if empty (default: []) + - `exclude_lines`: [Array] A list of regular expressions to match the files that you want to exclude. + Ignored if empty (default: []) + - `max_bytes`: [Integer] The maximum number of bytes that a single log message can have (default: 10485760) + - `json`: [Hash] Options that control how filebeat handles decoding of log messages in JSON format + [See above](#json-logs). (default: {}) + - `multiline`: [Hash] Options that control how Filebeat handles log messages that span multiple lines. + [See above](#multiline-logs). (default: {}) + +## Limitations +This module doesn't load the [elasticsearch index template](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html#filebeat-template) into elasticsearch (required when shipping +directly to elasticsearch). + +When installing on Windows, there's an expectation that `C:\Temp` already exists, or an alternative +location specified in the `tmp_dir` parameter exists and is writable by puppet. The temp directory +is used to store the downloaded installer only. + +### Generic template + +By default, a generic, open ended template is used that simply converts your configuration into +a hash that is produced as YAML on the system. To use a template that is more strict, but possibly +incomplete, set `conf_template` to `filebeat/filebeat.yml.erb`. + +### Registry Path + +The default registry file in this module doesn't match the filebeat default, but moving the file +while the filbeat service is running can cause data duplication or data loss. If you're installing +filebeat for the first time you should consider setting `registry_file` to match the +[default](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-global-options.html#_registry_file). + +Be sure to include a path or the file will be put at the root of your filesystem. + +### Debian Systems + +Filebeat 5.x and newer requires apt-transport-https, but this module won't install it for you. + +### Using config_file +There are a few very specific use cases where you don't want this module to directly manage the filebeat +configuration file, but you still want the configuration file on the system at a different location. +Setting `config_file` will write the filebeat configuration file to an alternate location, but it will not +update the init script. If you don't also manage the correct file (/etc/filebeat/filebeat.yml on Linux, +C:/Program Files/Filebeat/filebeat.yml on Windows) then filebeat won't be able to start. + +If you're copying the alternate config file location into the real location you'll need to include some +metaparameters like +```puppet +file { '/etc/filebeat/filebeat.yml': + ensure => file, + source => 'file:///etc/filebeat/filebeat.special', + require => File['filebeat.yml'], + notify => Service['filebeat'], +} +``` +to ensure that services are managed like you might expect. + +## Development + +Pull requests and bug reports are welcome. If you're sending a pull request, please consider +writing tests if applicable. diff --git a/Rakefile b/Rakefile new file mode 100644 index 00000000..204fb18c --- /dev/null +++ b/Rakefile @@ -0,0 +1,75 @@ +require 'puppetlabs_spec_helper/rake_tasks' +require 'puppet-syntax/tasks/puppet-syntax' +require 'puppet_blacksmith/rake_tasks' if Bundler.rubygems.find_name('puppet-blacksmith').any? +require 'github_changelog_generator/task' if Bundler.rubygems.find_name('github_changelog_generator').any? + +def changelog_user + return unless Rake.application.top_level_tasks.include? "changelog" + returnVal = nil || JSON.load(File.read('metadata.json'))['author'] + raise "unable to find the changelog_user in .sync.yml, or the author in metadata.json" if returnVal.nil? + puts "GitHubChangelogGenerator user:#{returnVal}" + returnVal +end + +def changelog_project + return unless Rake.application.top_level_tasks.include? "changelog" + returnVal = nil || JSON.load(File.read('metadata.json'))['name'] + raise "unable to find the changelog_project in .sync.yml or the name in metadata.json" if returnVal.nil? + puts "GitHubChangelogGenerator project:#{returnVal}" + returnVal +end + +def changelog_future_release + return unless Rake.application.top_level_tasks.include? "changelog" + returnVal = JSON.load(File.read('metadata.json'))['version'] + raise "unable to find the future_release (version) in metadata.json" if returnVal.nil? + puts "GitHubChangelogGenerator future_release:#{returnVal}" + returnVal +end + +PuppetLint.configuration.send('disable_relative') + +if Bundler.rubygems.find_name('github_changelog_generator').any? + GitHubChangelogGenerator::RakeTask.new :changelog do |config| + raise "Set CHANGELOG_GITHUB_TOKEN environment variable eg 'export CHANGELOG_GITHUB_TOKEN=valid_token_here'" if Rake.application.top_level_tasks.include? "changelog" and ENV['CHANGELOG_GITHUB_TOKEN'].nil? + config.user = "#{changelog_user}" + config.project = "#{changelog_project}" + config.future_release = "#{changelog_future_release}" + config.exclude_labels = ['maintenance'] + config.header = "# Change log\n\nAll notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org)." + config.add_pr_wo_labels = true + config.issues = false + config.merge_prefix = "### UNCATEGORIZED PRS; GO LABEL THEM" + config.configure_sections = { + "Changed" => { + "prefix" => "### Changed", + "labels" => ["backwards-incompatible"], + }, + "Added" => { + "prefix" => "### Added", + "labels" => ["feature", "enhancement"], + }, + "Fixed" => { + "prefix" => "### Fixed", + "labels" => ["bugfix"], + }, + } + end +else + desc 'Generate a Changelog from GitHub' + task :changelog do + raise <= Gem::Version.new('2.2.2')" +EOM + end +end + diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..bb608642 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,60 @@ +--- +version: 1.1.x.{build} +skip_commits: + message: /^\(?doc\)?.*/ +clone_depth: 10 +init: + - SET + - 'mkdir C:\ProgramData\PuppetLabs\code && exit 0' + - 'mkdir C:\ProgramData\PuppetLabs\facter && exit 0' + - 'mkdir C:\ProgramData\PuppetLabs\hiera && exit 0' + - 'mkdir C:\ProgramData\PuppetLabs\puppet\var && exit 0' +environment: + matrix: + - + RUBY_VERSION: 24-x64 + CHECK: syntax lint metadata_lint check:symlinks check:git_ignore check:dot_underscore check:test_file rubocop + - + PUPPET_GEM_VERSION: ~> 4.0 + RUBY_VERSION: 21 + CHECK: parallel_spec + - + PUPPET_GEM_VERSION: ~> 4.0 + RUBY_VERSION: 21-x64 + CHECK: parallel_spec + - + PUPPET_GEM_VERSION: ~> 5.0 + RUBY_VERSION: 24 + CHECK: parallel_spec + - + PUPPET_GEM_VERSION: ~> 5.0 + RUBY_VERSION: 24-x64 + CHECK: parallel_spec + - + PUPPET_GEM_VERSION: ~> 6.0 + RUBY_VERSION: 25 + CHECK: parallel_spec + - + PUPPET_GEM_VERSION: ~> 6.0 + RUBY_VERSION: 25-x64 + CHECK: parallel_spec +matrix: + fast_finish: true +install: + - set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH% + - bundle install --jobs 4 --retry 2 --without system_tests + - type Gemfile.lock +build: off +test_script: + - bundle exec puppet -V + - ruby -v + - gem -v + - bundle -v + - bundle exec rake %CHECK% +notifications: + - provider: Email + to: + - nobody@nowhere.com + on_build_success: false + on_build_failure: false + on_build_status_changed: false diff --git a/lib/facter/filebeat_version.rb b/lib/facter/filebeat_version.rb new file mode 100644 index 00000000..db45f803 --- /dev/null +++ b/lib/facter/filebeat_version.rb @@ -0,0 +1,18 @@ +require 'facter' +Facter.add('filebeat_version') do + confine 'kernel' => ['FreeBSD', 'OpenBSD', 'Linux', 'Windows'] + if File.executable?('/usr/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/bin/filebeat version') + elsif File.executable?('/usr/local/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/local/bin/filebeat --version') + elsif File.executable?('/usr/share/filebeat/bin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/share/filebeat/bin/filebeat --version') + elsif File.executable?('/usr/local/sbin/filebeat') + filebeat_version = Facter::Util::Resolution.exec('/usr/local/sbin/filebeat --version') + elsif File.exist?('c:\Program Files\Filebeat\filebeat.exe') + filebeat_version = Facter::Util::Resolution.exec('"c:\Program Files\Filebeat\filebeat.exe" --version') + end + setcode do + filebeat_version.nil? ? false : %r{^filebeat version ([^\s]+)?}.match(filebeat_version)[1] + end +end diff --git a/manifests/config.pp b/manifests/config.pp new file mode 100644 index 00000000..4a4676e2 --- /dev/null +++ b/manifests/config.pp @@ -0,0 +1,211 @@ +# filebeat::config +# +# Manage the configuration files for filebeat +# +# @summary A private class to manage the filebeat config file +class filebeat::config { + $major_version = $filebeat::major_version + + if versioncmp($major_version, '6') >= 0 { + $filebeat_config_temp = delete_undef_values({ + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'name' => $filebeat::beat_name, + 'tags' => $filebeat::tags, + 'max_procs' => $filebeat::max_procs, + 'fields' => $filebeat::fields, + 'fields_under_root' => $filebeat::fields_under_root, + 'filebeat' => { + 'registry_file' => $filebeat::registry_file, + 'config.prospectors' => { + 'enabled' => true, + 'path' => "${filebeat::config_dir}/*.yml", + }, + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'modules' => $filebeat::modules, + }, + 'output' => $filebeat::outputs, + 'shipper' => $filebeat::shipper, + 'logging' => $filebeat::logging, + 'runoptions' => $filebeat::run_options, + 'processors' => $filebeat::processors, + 'setup' => $filebeat::setup, + }) + # Add the 'xpack' section if supported (version >= 6.1.0) and not undef + if $filebeat::xpack and versioncmp($filebeat::package_ensure, '6.1.0') >= 0 { + $filebeat_config = deep_merge($filebeat_config_temp, {'xpack' => $filebeat::xpack}) + } + else { + $filebeat_config = $filebeat_config_temp + } + } else { + $filebeat_config_temp = delete_undef_values({ + 'shutdown_timeout' => $filebeat::shutdown_timeout, + 'name' => $filebeat::beat_name, + 'tags' => $filebeat::tags, + 'queue_size' => $filebeat::queue_size, + 'max_procs' => $filebeat::max_procs, + 'fields' => $filebeat::fields, + 'fields_under_root' => $filebeat::fields_under_root, + 'filebeat' => { + 'spool_size' => $filebeat::spool_size, + 'idle_timeout' => $filebeat::idle_timeout, + 'registry_file' => $filebeat::registry_file, + 'publish_async' => $filebeat::publish_async, + 'config_dir' => $filebeat::config_dir, + 'shutdown_timeout' => $filebeat::shutdown_timeout, + }, + 'output' => $filebeat::outputs, + 'shipper' => $filebeat::shipper, + 'logging' => $filebeat::logging, + 'runoptions' => $filebeat::run_options, + 'processors' => $filebeat::processors, + }) + # Add the 'modules' section if supported (version >= 5.2.0) + if versioncmp($filebeat::package_ensure, '5.2.0') >= 0 { + $filebeat_config = deep_merge($filebeat_config_temp, {'modules' => $filebeat::modules}) + } + else { + $filebeat_config = $filebeat_config_temp + } + } + + if $::filebeat_version { + $skip_validation = versioncmp($::filebeat_version, $filebeat::major_version) ? { + -1 => true, + default => false, + } + } else { + $skip_validation = false + } + + Filebeat::Prospector <| |> -> File['filebeat.yml'] + + case $::kernel { + 'Linux' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $major_version ? { + '5' => "${filebeat::filebeat_path} -N -configtest -c %", + default => "${filebeat::filebeat_path} -c % test config", + }, + } + + file {'filebeat.yml': + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + owner => $filebeat::config_file_owner, + group => $filebeat::config_file_group, + mode => $filebeat::config_file_mode, + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + + file {'filebeat-config-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::config_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + } + } # end Linux + + 'FreeBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => '/usr/local/sbin/filebeat -N -configtest -c %', + } + + file {'filebeat.yml': + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + owner => $filebeat::config_file_owner, + group => $filebeat::config_file_group, + mode => $filebeat::config_file_mode, + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + + file {'filebeat-config-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::config_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + } + } # end FreeBSD + + 'OpenBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $major_version ? { + '5' => "${filebeat::filebeat_path} -N -configtest -c %", + default => "${filebeat::filebeat_path} -c % test config", + }, + } + + file {'filebeat.yml': + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + owner => $filebeat::config_file_owner, + group => $filebeat::config_file_group, + mode => $filebeat::config_file_mode, + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + + file {'filebeat-config-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::config_dir, + owner => $filebeat::config_dir_owner, + group => $filebeat::config_dir_group, + mode => $filebeat::config_dir_mode, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + } + } # end OpenBSD + + 'Windows' : { + $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') + $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') + + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => "\"${filebeat_path}\" -N -configtest -c \"%\"", + } + + file {'filebeat.yml': + ensure => $filebeat::file_ensure, + path => $filebeat::config_file, + content => template($filebeat::conf_template), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + require => File['filebeat-config-dir'], + } + + file {'filebeat-config-dir': + ensure => $filebeat::directory_ensure, + path => $filebeat::config_dir, + recurse => $filebeat::purge_conf_dir, + purge => $filebeat::purge_conf_dir, + force => true, + } + } # end Windows + + default : { + fail($filebeat::kernel_fail_message) + } + } +} diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100644 index 00000000..133db946 --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,142 @@ +# This class installs the Elastic filebeat log shipper and +# helps manage which files are shipped +# +# @example +# class { 'filebeat': +# outputs => { +# 'logstash' => { +# 'hosts' => [ +# 'localhost:5044', +# ], +# }, +# }, +# } +# +# @param package_ensure [String] The ensure parameter for the filebeat package (default: present) +# @param manage_repo [Boolean] Whether or not the upstream (elastic) repo should be configured or not (default: true) +# @param manage_apt [Boolean] Whether or not the apt class should be explicitly called or not (default: true) +# @param major_version [Enum] The major version of Filebeat to be installed. +# @param service_ensure [String] The ensure parameter on the filebeat service (default: running) +# @param service_enable [String] The enable parameter on the filebeat service (default: true) +# @param repo_priority [Integer] Repository priority. yum and apt supported (default: undef) +# @param spool_size [Integer] How large the spool should grow before being flushed to the network (default: 2048) +# @param idle_timeout [String] How often the spooler should be flushed even if spool size isn't reached (default: 5s) +# @param publish_async [Boolean] If set to true filebeat will publish while preparing the next batch of lines to send (defualt: false) +# @param registry_file [String] The registry file used to store positions, absolute or relative to working directory (default .filebeat) +# @param config_dir [String] The directory where prospectors should be defined (default: /etc/filebeat/conf.d) +# @param config_dir_mode [String] The unix permissions mode set on the configuration directory (default: 0755) +# @param config_file_mode [String] The unix permissions mode set on configuration files (default: 0644) +# @param purge_conf_dir [Boolean] Should files in the prospector configuration directory not managed by puppet be automatically purged +# @param outputs [Hash] Will be converted to YAML for the required outputs section of the configuration (see documentation, and above) +# @param shipper [Hash] Will be converted to YAML to create the optional shipper section of the filebeat config (see documentation) +# @param logging [Hash] Will be converted to YAML to create the optional logging section of the filebeat config (see documentation) +# @param modules [Array] Will be converted to YAML to create the optional modules section of the filebeat config (see documentation) +# @param conf_template [String] The configuration template to use to generate the main filebeat.yml config file +# @param download_url [String] The URL of the zip file that should be downloaded to install filebeat (windows only) +# @param install_dir [String] Where filebeat should be installed (windows only) +# @param tmp_dir [String] Where filebeat should be temporarily downloaded to so it can be installed (windows only) +# @param shutdown_timeout [String] How long filebeat waits on shutdown for the publisher to finish sending events +# @param beat_name [String] The name of the beat shipper (default: hostname) +# @param tags [Array] A list of tags that will be included with each published transaction +# @param queue_size [String] The internal queue size for events in the pipeline +# @param max_procs [Integer] The maximum number of CPUs that can be simultaneously used +# @param fields [Hash] Optional fields that should be added to each event output +# @param fields_under_root [Boolean] If set to true, custom fields are stored in the top level instead of under fields +# @param processors [Array] Processors that will be added. Commonly used to create processors using hiera. +# @param prospectors [Hash] Prospectors that will be created. Commonly used to create prospectors using hiera +# @param setup [Hash] setup that will be created. Commonly used to create setup using hiera +# @param prospectors_merge [Boolean] Whether $prospectors should merge all hiera sources, or use simple automatic parameter lookup +# proxy_address [String] Proxy server to use for downloading files +# @param xpack [Hash] Configuration items to export internal stats to a monitoring Elasticsearch cluster +class filebeat ( + String $package_ensure = $filebeat::params::package_ensure, + Boolean $manage_repo = $filebeat::params::manage_repo, + Boolean $manage_apt = $filebeat::params::manage_apt, + Enum['5','6'] $major_version = $filebeat::params::major_version, + Variant[Boolean, Enum['stopped', 'running']] $service_ensure = $filebeat::params::service_ensure, + Boolean $service_enable = $filebeat::params::service_enable, + Optional[String] $service_provider = $filebeat::params::service_provider, + Optional[Integer] $repo_priority = undef, + Integer $spool_size = $filebeat::params::spool_size, + String $idle_timeout = $filebeat::params::idle_timeout, + Boolean $publish_async = $filebeat::params::publish_async, + String $registry_file = $filebeat::params::registry_file, + String $config_file = $filebeat::params::config_file, + Optional[String] $config_file_owner = $filebeat::params::config_file_owner, + Optional[String] $config_file_group = $filebeat::params::config_file_group, + String[4,4] $config_dir_mode = $filebeat::params::config_dir_mode, + String $config_dir = $filebeat::params::config_dir, + String[4,4] $config_file_mode = $filebeat::params::config_file_mode, + Optional[String] $config_dir_owner = $filebeat::params::config_dir_owner, + Optional[String] $config_dir_group = $filebeat::params::config_dir_group, + Boolean $purge_conf_dir = $filebeat::params::purge_conf_dir, + Hash $outputs = $filebeat::params::outputs, + Hash $shipper = $filebeat::params::shipper, + Hash $logging = $filebeat::params::logging, + Hash $run_options = $filebeat::params::run_options, + String $conf_template = $filebeat::params::conf_template, + Optional[Variant[Stdlib::HTTPUrl, Stdlib::HTTPSUrl]] $download_url = undef, # lint:ignore:140chars + Optional[String] $install_dir = $filebeat::params::install_dir, + String $tmp_dir = $filebeat::params::tmp_dir, + String $shutdown_timeout = $filebeat::params::shutdown_timeout, + String $beat_name = $filebeat::params::beat_name, + Array $tags = $filebeat::params::tags, + Integer $queue_size = $filebeat::params::queue_size, + Optional[Integer] $max_procs = $filebeat::params::max_procs, + Hash $fields = $filebeat::params::fields, + Boolean $fields_under_root = $filebeat::params::fields_under_root, + Boolean $disable_config_test = $filebeat::params::disable_config_test, + Array $processors = [], + Hash $prospectors = {}, + Hash $setup = {}, + Array $modules = [], + Optional[Variant[Stdlib::HTTPUrl, Stdlib::HTTPSUrl]] $proxy_address = undef, # lint:ignore:140chars + Stdlib::Absolutepath $filebeat_path = $filebeat::params::filebeat_path, + Optional[Hash] $xpack = $filebeat::params::xpack, +) inherits filebeat::params { + + include ::stdlib + + $real_download_url = $download_url ? { + undef => "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${package_ensure}-windows-${filebeat::params::url_arch}.zip", + default => $download_url, + } + + if $config_file != $filebeat::params::config_file { + warning('You\'ve specified a non-standard config_file location - filebeat may fail to start unless you\'re doing something to fix this') + } + + if $package_ensure == 'absent' { + $alternate_ensure = 'absent' + $real_service_ensure = 'stopped' + $file_ensure = 'absent' + $directory_ensure = 'absent' + } else { + $alternate_ensure = 'present' + $file_ensure = 'file' + $directory_ensure = 'directory' + $real_service_ensure = $service_ensure + } + + # If we're removing filebeat, do things in a different order to make sure + # we remove as much as possible + if $package_ensure == 'absent' { + anchor { 'filebeat::begin': } + -> class { '::filebeat::config': } + -> class { '::filebeat::install': } + -> class { '::filebeat::service': } + -> anchor { 'filebeat::end': } + } else { + anchor { 'filebeat::begin': } + -> class { '::filebeat::install': } + -> class { '::filebeat::config': } + -> class { '::filebeat::service': } + -> anchor { 'filebeat::end': } + } + + if $package_ensure != 'absent' { + if !empty($prospectors) { + create_resources('filebeat::prospector', $prospectors) + } + } +} diff --git a/manifests/install.pp b/manifests/install.pp new file mode 100644 index 00000000..fd65f4e7 --- /dev/null +++ b/manifests/install.pp @@ -0,0 +1,43 @@ +# filebeat::install +# +# A private class to manage the installation of Filebeat +# +# @summary A private class that manages the install of Filebeat +class filebeat::install { + anchor { 'filebeat::install::begin': } + + case $::kernel { + 'Linux': { + class{ '::filebeat::install::linux': + notify => Class['filebeat::service'], + } + Anchor['filebeat::install::begin'] -> Class['filebeat::install::linux'] -> Anchor['filebeat::install::end'] + if $::filebeat::manage_repo { + class { '::filebeat::repo': } + Class['filebeat::repo'] -> Class['filebeat::install::linux'] + } + } + 'FreeBSD': { + class{ '::filebeat::install::freebsd': + notify => Class['filebeat::service'], + } + Anchor['filebeat::install::begin'] -> Class['filebeat::install::freebsd'] -> Anchor['filebeat::install::end'] + } + 'OpenBSD': { + class{'filebeat::install::openbsd':} + Anchor['filebeat::install::begin'] -> Class['filebeat::install::openbsd'] -> Anchor['filebeat::install::end'] + } + 'Windows': { + class{'::filebeat::install::windows': + notify => Class['filebeat::service'], + } + Anchor['filebeat::install::begin'] -> Class['filebeat::install::windows'] -> Anchor['filebeat::install::end'] + } + default: { + fail($filebeat::kernel_fail_message) + } + } + + anchor { 'filebeat::install::end': } + +} diff --git a/manifests/install/freebsd.pp b/manifests/install/freebsd.pp new file mode 100644 index 00000000..e67ee8ef --- /dev/null +++ b/manifests/install/freebsd.pp @@ -0,0 +1,13 @@ +# filebeat::install::freebsd +# +# Install the FreeBSD filebeat package +# +# @summary A simple class to install the filebeat package +# +class filebeat::install::freebsd { + + # filebeat, heartbeat, metricbeat, packetbeat are all contained in a + # single FreeBSD Package (see https://www.freshports.org/sysutils/beats/ ) + ensure_packages (['beats'], {ensure => $filebeat::package_ensure}) + +} diff --git a/manifests/install/linux.pp b/manifests/install/linux.pp new file mode 100644 index 00000000..fd54afcd --- /dev/null +++ b/manifests/install/linux.pp @@ -0,0 +1,15 @@ +# filebeat::install::linux +# +# Install the linux filebeat package +# +# @summary A simple class to install the filebeat package +# +class filebeat::install::linux { + if $::kernel != 'Linux' { + fail('filebeat::install::linux shouldn\'t run on Windows') + } + + package {'filebeat': + ensure => $filebeat::package_ensure, + } +} diff --git a/manifests/install/openbsd.pp b/manifests/install/openbsd.pp new file mode 100644 index 00000000..07f0e1f9 --- /dev/null +++ b/manifests/install/openbsd.pp @@ -0,0 +1,6 @@ +# to manage filebeat installation on OpenBSD +class filebeat::install::openbsd { + package {'filebeat': + ensure => $filebeat::package_ensure, + } +} diff --git a/manifests/install/windows.pp b/manifests/install/windows.pp new file mode 100644 index 00000000..7b831455 --- /dev/null +++ b/manifests/install/windows.pp @@ -0,0 +1,82 @@ +# filebeat::install::windows +# +# Download and install filebeat on Windows +# +# @summary A private class that installs filebeat on Windows +# +class filebeat::install::windows { + # I'd like to use chocolatey to do this install, but the package for chocolatey is + # failing for updates and seems rather unpredictable at the moment. We may revisit + # that in the future as it would greatly simplify this code and basically reduce it to + # one package resource with type => chocolatey.... + + $filename = regsubst($filebeat::real_download_url, '^https?.*\/([^\/]+)\.[^.].*', '\1') + $foldername = 'Filebeat' + $zip_file = join([$filebeat::tmp_dir, "${filename}.zip"], '/') + $install_folder = join([$filebeat::install_dir, $foldername], '/') + $version_file = join([$install_folder, $filename], '/') + + Exec { + provider => powershell, + } + + if ! defined(File[$filebeat::install_dir]) { + file { $filebeat::install_dir: + ensure => directory, + } + } + + # Note: We can use archive for unzip and cleanup, thus removing the following two resources. + # However, this requires 7zip, which archive can install via chocolatey: + # https://github.com/voxpupuli/puppet-archive/blob/master/manifests/init.pp#L31 + # I'm not choosing to impose those dependencies on anyone at this time... + archive { $zip_file: + source => $filebeat::real_download_url, + cleanup => false, + creates => $version_file, + proxy_server => $filebeat::proxy_address, + } + + exec { "unzip ${filename}": + command => "\$sh=New-Object -COM Shell.Application;\$sh.namespace((Convert-Path '${filebeat::install_dir}')).Copyhere(\$sh.namespace((Convert-Path '${zip_file}')).items(), 16)", # lint:ignore:140chars + creates => $version_file, + require => [ + File[$filebeat::install_dir], + Archive[$zip_file], + ], + } + + # Clean up after ourselves + file { $zip_file: + ensure => absent, + backup => false, + require => Exec["unzip ${filename}"], + } + + # You can't remove the old dir while the service has files locked... + exec { "stop service ${filename}": + command => 'Set-Service -Name filebeat -Status Stopped', + creates => $version_file, + onlyif => 'if(Get-WmiObject -Class Win32_Service -Filter "Name=\'filebeat\'") {exit 0} else {exit 1}', + require => Exec["unzip ${filename}"], + } + + exec { "rename ${filename}": + command => "Remove-Item '${install_folder}' -Recurse -Force -ErrorAction SilentlyContinue;Rename-Item '${filebeat::install_dir}/${filename}' '${install_folder}'", # lint:ignore:140chars + creates => $version_file, + require => Exec["stop service ${filename}"], + } + + exec { "mark ${filename}": + command => "New-Item '${version_file}' -ItemType file", + creates => $version_file, + require => Exec["rename ${filename}"], + } + + exec { "install ${filename}": + cwd => $install_folder, + command => './install-service-filebeat.ps1', + refreshonly => true, + subscribe => Exec["mark ${filename}"], + } +} diff --git a/manifests/params.pp b/manifests/params.pp new file mode 100644 index 00000000..386b7f33 --- /dev/null +++ b/manifests/params.pp @@ -0,0 +1,145 @@ +# filebeat::params +# +# Set a number of default parameters +# +# @summary Set a bunch of default parameters +class filebeat::params { + $service_ensure = running + $service_enable = true + $spool_size = 2048 + $idle_timeout = '5s' + $publish_async = false + $shutdown_timeout = '0' + $beat_name = $::fqdn + $tags = [] + $queue_size = 1000 + $max_procs = undef + $config_file_mode = '0644' + $config_dir_mode = '0755' + $purge_conf_dir = true + $fields = {} + $fields_under_root = false + $outputs = {} + $shipper = {} + $logging = {} + $run_options = {} + $modules = [] + $kernel_fail_message = "${::kernel} is not supported by filebeat." + $osfamily_fail_message = "${::osfamily} is not supported by filebeat." + $conf_template = "${module_name}/pure_hash.yml.erb" + $disable_config_test = false + $xpack = undef + + # These are irrelevant as long as the template is set based on the major_version parameter + # if versioncmp('1.9.1', $::rubyversion) > 0 { + # $conf_template = "${module_name}/filebeat.yml.ruby18.erb" + # } else { + # $conf_template = "${module_name}/filebeat.yml.erb" + # } + # + + # Archlinux and OpenBSD have proper packages in the official repos + # we shouldn't manage the repo on them + case $facts['os']['family'] { + 'Archlinux': { + $manage_repo = false + $manage_apt = false + $filebeat_path = '/usr/bin/filebeat' + $major_version = '6' + } + 'OpenBSD': { + $manage_repo = false + $manage_apt = false + $filebeat_path = '/usr/local/bin/filebeat' + # lint:ignore:only_variable_string + $major_version = versioncmp('6.3', $::kernelversion) < 0 ? { + # lint:endignore + true => '6', + default => '5' + } + } + default: { + $manage_repo = true + $manage_apt = true + $filebeat_path = '/usr/share/filebeat/bin/filebeat' + $major_version = '6' + } + } + case $::kernel { + 'Linux' : { + $package_ensure = present + $config_file = '/etc/filebeat/filebeat.yml' + $config_dir = '/etc/filebeat/conf.d' + $config_file_owner = 'root' + $config_file_group = 'root' + $config_dir_owner = 'root' + $config_dir_group = 'root' + $registry_file = '/var/lib/filebeat/registry' + # These parameters are ignored if/until tarball installs are supported in Linux + $tmp_dir = '/tmp' + $install_dir = undef + case $::osfamily { + 'RedHat': { + $service_provider = 'redhat' + } + default: { + $service_provider = undef + } + } + $url_arch = undef + } + + 'FreeBSD': { + $package_ensure = present + $config_file = '/usr/local/etc/filebeat.yml' + $config_dir = '/usr/local/etc/filebeat.d' + $config_file_owner = 'root' + $config_file_group = 'wheel' + $config_dir_owner = 'root' + $config_dir_group = 'wheel' + $registry_file = '/var/lib/filebeat/registry' + $tmp_dir = '/tmp' + $service_provider = undef + $install_dir = undef + $url_arch = undef + } + + 'OpenBSD': { + $package_ensure = present + $config_file = '/etc/filebeat/filebeat.yml' + $config_dir = '/etc/filebeat/conf.d' + $config_file_owner = 'root' + $config_file_group = 'wheel' + $config_dir_owner = 'root' + $config_dir_group = 'wheel' + $registry_file = '/var/db/filebeat/.filebeat' + $tmp_dir = '/tmp' + $service_provider = undef + $install_dir = undef + $url_arch = undef + } + + 'Windows' : { + $package_ensure = '5.6.2' + $config_file_owner = 'Administrator' + $config_file_group = undef + $config_dir_owner = 'Administrator' + $config_dir_group = undef + $config_file = 'C:/Program Files/Filebeat/filebeat.yml' + $config_dir = 'C:/Program Files/Filebeat/conf.d' + $registry_file = 'C:/ProgramData/filebeat/registry' + $install_dir = 'C:/Program Files' + $tmp_dir = 'C:/Windows/Temp' + $service_provider = undef + $url_arch = $::architecture ? { + 'x86' => 'x86', + 'x64' => 'x86_64', + default => fail("${::architecture} is not supported by filebeat."), + } + } + + default : { + fail($kernel_fail_message) + } + } +} diff --git a/manifests/prospector.pp b/manifests/prospector.pp new file mode 100644 index 00000000..4b8b879b --- /dev/null +++ b/manifests/prospector.pp @@ -0,0 +1,132 @@ +# filebeat::prospector +# +# A description of what this defined type does +# +# @summary A short summary of the purpose of this defined type. +# +# @example +# filebeat::prospector { 'namevar': } +define filebeat::prospector ( + Enum['absent', 'present'] $ensure = present, + Array[String] $paths = [], + Array[String] $exclude_files = [], + Array[String] $containers_ids = ['\'*\''], + String $containers_path = '/var/lib/docker/containers', + String $containers_stream = 'all', + Boolean $combine_partial = false, + Boolean $cri_parse_flags = false, + String $encoding = 'plain', + String $input_type = 'log', + Hash $fields = {}, + Boolean $fields_under_root = false, + Optional[String] $ignore_older = undef, + Optional[String] $close_older = undef, + String $doc_type = 'log', + String $scan_frequency = '10s', + Integer $harvester_buffer_size = 16384, + Optional[Integer] $harvester_limit = undef, + Boolean $tail_files = false, + String $backoff = '1s', + String $max_backoff = '10s', + Integer $backoff_factor = 2, + String $close_inactive = '5m', + Boolean $close_renamed = false, + Boolean $close_removed = true, + Boolean $close_eof = false, + Integer $clean_inactive = 0, + Boolean $clean_removed = true, + Integer $close_timeout = 0, + Boolean $force_close_files = false, + Array[String] $include_lines = [], + Array[String] $exclude_lines = [], + String $max_bytes = '10485760', + Hash $multiline = {}, + Hash $json = {}, + Array[String] $tags = [], + Boolean $symlinks = false, + Optional[String] $pipeline = undef, + Array $processors = [], +) { + + $prospector_template = $filebeat::major_version ? { + '5' => 'prospector5.yml.erb', + default => 'prospector.yml.erb', + } + + if $::filebeat_version { + $skip_validation = versioncmp($::filebeat_version, $filebeat::major_version) ? { + -1 => true, + default => false, + } + } else { + $skip_validation = false + } + + case $::kernel { + 'Linux', 'OpenBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $filebeat::major_version ? { + '5' => "\"${filebeat::filebeat_path}\" -N -configtest -c \"%\"", + default => "\"${filebeat::filebeat_path}\" -c \"${filebeat::config_file}\" test config", + }, + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => '0', + mode => $::filebeat::config_file_mode, + content => template("${module_name}/${prospector_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + before => File['filebeat.yml'], + } + } + + 'FreeBSD' : { + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => '/usr/local/sbin/filebeat -N -configtest -c %', + } + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + owner => 'root', + group => 'wheel', + mode => $::filebeat::config_file_mode, + content => template("${module_name}/${prospector_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + before => File['filebeat.yml'], + } + } + + 'Windows' : { + $cmd_install_dir = regsubst($filebeat::install_dir, '/', '\\', 'G') + $filebeat_path = join([$cmd_install_dir, 'Filebeat', 'filebeat.exe'], '\\') + + $validate_cmd = ($filebeat::disable_config_test or $skip_validation) ? { + true => undef, + default => $::filebeat_version ? { + '5' => "\"${filebeat_path}\" -N -configtest -c \"%\"", + default => "\"${filebeat_path}\" -c \"${filebeat::config_file}\" test config", + }, + } + + file { "filebeat-${name}": + ensure => $ensure, + path => "${filebeat::config_dir}/${name}.yml", + content => template("${module_name}/${prospector_template}"), + validate_cmd => $validate_cmd, + notify => Service['filebeat'], + before => File['filebeat.yml'], + } + } + + default : { + fail($filebeat::kernel_fail_message) + } + + } +} diff --git a/manifests/repo.pp b/manifests/repo.pp new file mode 100644 index 00000000..3d1a2163 --- /dev/null +++ b/manifests/repo.pp @@ -0,0 +1,69 @@ +# filebeat::repo +# +# Manage the repository for Filebeat (Linux only for now) +# +# @summary Manages the yum, apt, and zypp repositories for Filebeat +class filebeat::repo { + $debian_repo_url = "https://artifacts.elastic.co/packages/${filebeat::major_version}.x/apt" + $yum_repo_url = "https://artifacts.elastic.co/packages/${filebeat::major_version}.x/yum" + + case $::osfamily { + 'Debian': { + if $::filebeat::manage_apt == true { + include ::apt + } + + Class['apt::update'] -> Package['filebeat'] + + if !defined(Apt::Source['beats']){ + apt::source { 'beats': + ensure => $::filebeat::alternate_ensure, + location => $debian_repo_url, + release => 'stable', + repos => 'main', + pin => $::filebeat::repo_priority, + key => { + id => '46095ACC8548582C1A2699A9D27D666CD88E42B4', + source => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', + }, + } + } + } + 'RedHat', 'Linux': { + if !defined(Yumrepo['beats']){ + yumrepo { 'beats': + ensure => $::filebeat::alternate_ensure, + descr => 'elastic beats repo', + baseurl => $yum_repo_url, + gpgcheck => 1, + gpgkey => 'https://artifacts.elastic.co/GPG-KEY-elasticsearch', + priority => $::filebeat::repo_priority, + enabled => 1, + } + } + } + 'Suse': { + exec { 'topbeat_suse_import_gpg': + command => 'rpmkeys --import https://artifacts.elastic.co/GPG-KEY-elasticsearch', + unless => 'test $(rpm -qa gpg-pubkey | grep -i "D88E42B4" | wc -l) -eq 1 ', + notify => [ Zypprepo['beats'] ], + } + if !defined(Zypprepo['beats']){ + zypprepo { 'beats': + ensure => $::filebeat::alternate_ensure, + baseurl => $yum_repo_url, + enabled => 1, + autorefresh => 1, + name => 'beats', + gpgcheck => 1, + gpgkey => 'https://packages.elastic.co/GPG-KEY-elasticsearch', + type => 'yum', + } + } + } + default: { + fail($filebeat::osfamily_fail_message) + } + } + +} diff --git a/manifests/service.pp b/manifests/service.pp new file mode 100644 index 00000000..80afd08f --- /dev/null +++ b/manifests/service.pp @@ -0,0 +1,12 @@ +# filebeat::service +# +# Manage the filebeat service +# +# @summary Manage the filebeat service +class filebeat::service { + service { 'filebeat': + ensure => $filebeat::real_service_ensure, + enable => $filebeat::service_enable, + provider => $filebeat::service_provider, + } +} diff --git a/metadata.json b/metadata.json new file mode 100644 index 00000000..382a4e99 --- /dev/null +++ b/metadata.json @@ -0,0 +1,108 @@ +{ + "name": "pcfens-filebeat", + "version": "3.3.3", + "author": "pcfens", + "summary": "A module to install and manage the filebeat log shipper", + "license": "Apache-2.0", + "source": "https://github.com/pcfens/puppet-filebeat", + "project_page": "https://github.com/pcfens/puppet-filebeat", + "issues_url": "https://github.com/pcfens/puppet-filebeat/issues", + "dependencies": [ + { + "name": "puppetlabs/stdlib", + "version_requirement": ">=4.13.0 < 6.0.0" + }, + { + "name": "puppetlabs/apt", + "version_requirement": ">=2.0.0 < 7.0.0" + }, + { + "name": "puppetlabs/powershell", + "version_requirement": ">= 1.0.1 < 3.0.0" + }, + { + "name": "puppet/archive", + "version_requirement": ">= 0.5.0 < 4.0.0" + } + ], + "operatingsystem_support": [ + { + "operatingsystem": "CentOS", + "operatingsystemrelease": [ + "5", + "6", + "7" + ] + }, + { + "operatingsystem": "OracleLinux", + "operatingsystemrelease": [ + "6", + "7" + ] + }, + { + "operatingsystem": "RedHat", + "operatingsystemrelease": [ + "5", + "6", + "7" + ] + }, + { + "operatingsystem": "Scientific", + "operatingsystemrelease": [ + "5", + "6", + "7" + ] + }, + { + "operatingsystem": "Debian", + "operatingsystemrelease": [ + "8" + ] + }, + { + "operatingsystem": "OpenBSD", + "operatingsystemrelease": [ + "6.1", + "6.2", + "6.3" + ] + }, + { + "operatingsystem": "Ubuntu", + "operatingsystemrelease": [ + "14.04", + "16.04", + "18.04" + ] + }, + { + "operatingsystem": "windows", + "operatingsystemrelease": [ + "2012", + "2012 R2" + ] + }, + { + "operatingsystem": "Archlinux" + } + ], + "requirements": [ + { + "name": "puppet", + "version_requirement": ">= 4.0.0 < 7.0.0" + } + ], + "tags": [ + "filebeat", + "logstash", + "elasticsearch", + "elastic" + ], + "pdk-version": "1.7.1", + "template-url": "file:///opt/puppetlabs/pdk/share/cache/pdk-templates.git", + "template-ref": "1.7.1-0-g810b982" +} diff --git a/spec/acceptance/001_basic_spec.rb b/spec/acceptance/001_basic_spec.rb new file mode 100644 index 00000000..8f2bcaa2 --- /dev/null +++ b/spec/acceptance/001_basic_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper_acceptance' + +RSpec.shared_examples 'filebeat' do + describe package('filebeat') do + it { is_expected.to be_installed } + end + + describe service('filebeat') do + it { is_expected.to be_enabled } + it { is_expected.to be_running } + end + + describe file('/etc/filebeat/filebeat.yml') do + it { is_expected.to be_file } + it { is_expected.to contain('---') } + it { is_expected.not_to contain('max_procs: !ruby') } + end +end + +describe 'filebeat class' do + let(:pp) do + <<-HEREDOC + if $::osfamily == 'Debian' { + include ::apt + + package { 'apt-transport-https': + ensure => present, + } + } + + class { 'filebeat': + major_version => '#{major_version}', + outputs => { + 'logstash' => { + 'bulk_max_size' => 1024, + 'hosts' => [ + 'localhost:5044', + ], + }, + 'file' => { + 'path' => '/tmp', + 'filename' => 'filebeat', + 'rotate_every_kb' => 10240, + 'number_of_files' => 2, + }, + }, + shipper => { + refresh_topology_freq => 10, + topology_expire => 15, + queue_size => 1000, + }, + logging => { + files => { + rotateeverybytes => 10485760, + keepfiles => 7, + } + }, + prospectors => { + 'system-logs' => { + doc_type => 'system', + paths => [ + '/var/log/dmesg', + ], + fields => { + service => 'system', + file => 'dmesg', + }, + tags => [ + 'tag1', + 'tag2', + 'tag3', + ] + } + } + } + HEREDOC + end + + context 'with $major_version = 5' do + let(:major_version) { 5 } + + it_behaves_like 'an idempotent resource' + include_examples 'filebeat' + end + + context 'with $major_version = 6' do + let(:major_version) { 6 } + + it_behaves_like 'an idempotent resource' + include_examples 'filebeat' + end +end diff --git a/spec/acceptance/nodesets/centos-7.yml b/spec/acceptance/nodesets/centos-7.yml new file mode 100644 index 00000000..6b5631e4 --- /dev/null +++ b/spec/acceptance/nodesets/centos-7.yml @@ -0,0 +1,16 @@ +HOSTS: + centos-7-x64: + roles: + - master + - database + platform: el-7-x86_64 + hypervisor: docker + image: centos:7 + docker_preserve_image: true + docker_cmd: '["/usr/sbin/init"]' + # install various tools required to get the image up to usable levels + docker_image_commands: + - 'yum install -y crontabs tar wget openssl sysvinit-tools iproute which initscripts' +CONFIG: + trace_limit: 200 + type: foss diff --git a/spec/acceptance/nodesets/default.yml b/spec/acceptance/nodesets/default.yml new file mode 100644 index 00000000..4203e73f --- /dev/null +++ b/spec/acceptance/nodesets/default.yml @@ -0,0 +1,15 @@ +HOSTS: + ubuntu-1604-x64: + roles: + - agent + platform: ubuntu-16.04-amd64 + hypervisor: docker + image: ubuntu:16.04 + docker_preserve_image: true + docker_cmd: '["/sbin/init"]' + docker_image_commands: + # ensure that upstart is booting correctly in the container + - 'rm /usr/sbin/policy-rc.d && rm /sbin/initctl && dpkg-divert --rename --remove /sbin/initctl && apt-get update && apt-get install -y net-tools wget locales && locale-gen en_US.UTF-8' +CONFIG: + trace_limit: 200 + type: foss diff --git a/spec/classes/config_spec.rb b/spec/classes/config_spec.rb new file mode 100644 index 00000000..89dcb9cf --- /dev/null +++ b/spec/classes/config_spec.rb @@ -0,0 +1,83 @@ +require 'spec_helper' + +describe 'filebeat::config' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}' }" } + + [5, 6].each do |version| + context "with $major_version == #{version}" do + let(:major_version) { version } + + let(:validate_cmd) do + path = case os_facts[:os]['family'] + when 'Archlinux' + '/usr/bin/filebeat' + else + '/usr/share/filebeat/bin/filebeat' + end + + case major_version + when 5 + "#{path} -N -configtest -c %" + else + "#{path} -c % test config" + end + end + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to compile } + it { + is_expected.to contain_file('filebeat.yml').with( + ensure: 'file', + path: '/etc/filebeat/filebeat.yml', + owner: 'root', + group: 'root', + mode: '0644', + validate_cmd: validate_cmd, + notify: 'Service[filebeat]', + require: 'File[filebeat-config-dir]', + ) + } + + it { + is_expected.to contain_file('filebeat-config-dir').with( + ensure: 'directory', + path: '/etc/filebeat/conf.d', + owner: 'root', + group: 'root', + mode: '0755', + recurse: true, + purge: true, + ) + } + when 'Windows' + it { + is_expected.to contain_file('filebeat.yml').with( + ensure: 'file', + path: 'C:/Program Files/Filebeat/filebeat.yml', + notify: 'Service[filebeat]', + require: 'File[filebeat-config-dir]', + ) + } + + it { + is_expected.to contain_file('filebeat-config-dir').with( + ensure: 'directory', + path: 'C:/Program Files/Filebeat/conf.d', + recurse: true, + purge: true, + ) + } + end + end + end + end + end +end diff --git a/spec/classes/coverage_spec.rb b/spec/classes/coverage_spec.rb new file mode 100644 index 00000000..c083569a --- /dev/null +++ b/spec/classes/coverage_spec.rb @@ -0,0 +1,3 @@ +require 'rspec-puppet' + +at_exit { RSpec::Puppet::Coverage.report! } diff --git a/spec/classes/filebeat_spec.rb b/spec/classes/filebeat_spec.rb new file mode 100644 index 00000000..dfecf6c4 --- /dev/null +++ b/spec/classes/filebeat_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe 'filebeat' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { is_expected.to contain_class('filebeat') } + it { is_expected.to contain_class('filebeat::params') } + it { is_expected.to contain_anchor('filebeat::begin') } + it { is_expected.to contain_anchor('filebeat::end') } + it { is_expected.to contain_class('filebeat::install') } + it { is_expected.to contain_class('filebeat::config') } + it { is_expected.to contain_class('filebeat::service') } + end + end +end diff --git a/spec/classes/install/linux_spec.rb b/spec/classes/install/linux_spec.rb new file mode 100644 index 00000000..fbf02ba0 --- /dev/null +++ b/spec/classes/install/linux_spec.rb @@ -0,0 +1,21 @@ +require 'spec_helper' + +describe 'filebeat::install::linux' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to compile } + it { is_expected.to contain_package('filebeat').with_ensure('present') } + else + it { is_expected.not_to compile } + end + end + end +end diff --git a/spec/classes/install/windows_spec.rb b/spec/classes/install/windows_spec.rb new file mode 100644 index 00000000..8dba7eaf --- /dev/null +++ b/spec/classes/install/windows_spec.rb @@ -0,0 +1,59 @@ +require 'spec_helper' + +describe 'filebeat::install::windows' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + case os_facts[:kernel] + when 'windows' + # it { is_expected.to compile } + it { is_expected.to contain_file('C:/Program Files').with_ensure('directory') } + it { + is_expected.to contain_archive('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + creates: 'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64', + ) + } + it { + is_expected.to contain_exec('install filebeat-5.6.2-windows-x86_64').with( + command: './install-service-filebeat.ps1', + ) + } + it { + is_expected.to contain_exec('unzip filebeat-5.6.2-windows-x86_64').with( + command: '$sh=New-Object -COM Shell.Application;$sh.namespace((Convert-Path \'C:/Program Files\')).'\ + 'Copyhere($sh.namespace((Convert-Path \'C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip\')).items(), 16)', + ) + } + it { + is_expected.to contain_exec('mark filebeat-5.6.2-windows-x86_64').with( + command: 'New-Item \'C:/Program Files/Filebeat/filebeat-5.6.2-windows-x86_64\' -ItemType file', + ) + } + it { + is_expected.to contain_exec('rename filebeat-5.6.2-windows-x86_64').with( + command: 'Remove-Item \'C:/Program Files/Filebeat\' -Recurse -Force -ErrorAction SilentlyContinue;'\ + 'Rename-Item \'C:/Program Files/filebeat-5.6.2-windows-x86_64\' \'C:/Program Files/Filebeat\'', + ) + } + it { + is_expected.to contain_exec('stop service filebeat-5.6.2-windows-x86_64').with( + command: 'Set-Service -Name filebeat -Status Stopped', + ) + } + it { + is_expected.to contain_file('C:/Windows/Temp/filebeat-5.6.2-windows-x86_64.zip').with( + ensure: 'absent', + ) + } + + else + it { is_expected.not_to compile } + end + end + end +end diff --git a/spec/classes/install_spec.rb b/spec/classes/install_spec.rb new file mode 100644 index 00000000..f8b18073 --- /dev/null +++ b/spec/classes/install_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe 'filebeat::install' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { is_expected.to contain_anchor('filebeat::install::begin') } + it { is_expected.to contain_anchor('filebeat::install::end') } + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to contain_class('filebeat::install::linux') } + it { is_expected.to contain_class('filebeat::repo') } unless os_facts[:os]['family'] == 'Archlinux' + it { is_expected.not_to contain_class('filebeat::install::windows') } + + when 'Windows' + it { is_expected.to contain_class('filebeat::install::windows') } + it { is_expected.not_to contain_class('filebeat::install::linux') } + end + end + end +end diff --git a/spec/classes/params_spec.rb b/spec/classes/params_spec.rb new file mode 100644 index 00000000..f509e153 --- /dev/null +++ b/spec/classes/params_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe 'filebeat::params' do + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + it { is_expected.to compile } + end + end +end diff --git a/spec/classes/repo_spec.rb b/spec/classes/repo_spec.rb new file mode 100644 index 00000000..75a59e40 --- /dev/null +++ b/spec/classes/repo_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe 'filebeat::repo' do + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + let(:pre_condition) { "class { 'filebeat': major_version => '#{major_version}' }" } + + [5, 6].each do |version| + context "with $major_version == #{version}" do + let(:major_version) { version } + + case os_facts[:kernel] + when 'Linux' + it { is_expected.to compile } unless os_facts[:os]['family'] == 'Archlinux' + case os_facts[:osfamily] + when 'Debian' + it { + is_expected.to contain_apt__source('beats').with( + ensure: 'present', + location: "https://artifacts.elastic.co/packages/#{major_version}.x/apt", + ) + } + when 'RedHat' + it { + is_expected.to contain_yumrepo('beats').with( + ensure: 'present', + baseurl: "https://artifacts.elastic.co/packages/#{major_version}.x/yum", + ) + } + end + else + it { is_expected.not_to compile } + end + end + end + end + end +end diff --git a/spec/classes/service_spec.rb b/spec/classes/service_spec.rb new file mode 100644 index 00000000..a8292355 --- /dev/null +++ b/spec/classes/service_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe 'filebeat::service' do + let :pre_condition do + 'include ::filebeat' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { + is_expected.to contain_service('filebeat').with( + ensure: 'running', + enable: true, + ) + } + end + end +end diff --git a/spec/default_facts.yml b/spec/default_facts.yml new file mode 100644 index 00000000..cd40f98c --- /dev/null +++ b/spec/default_facts.yml @@ -0,0 +1,9 @@ +# Use default_module_facts.yml for module specific facts. +# +# Facts specified here will override the values provided by rspec-puppet-facts. +--- +concat_basedir: "/tmp" +ipaddress: "172.16.254.254" +is_pe: false +macaddress: "AA:AA:AA:AA:AA:AA" +filebeat_version: undef diff --git a/spec/defines/prospector_spec.rb b/spec/defines/prospector_spec.rb new file mode 100644 index 00000000..308da7dd --- /dev/null +++ b/spec/defines/prospector_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +describe 'filebeat::prospector' do + let :pre_condition do + 'class { "filebeat": + outputs => { + "logstash" => { + "hosts" => [ + "localhost:5044", + ], + }, + }, + }' + end + + on_supported_os(facterversion: '2.4').each do |os, os_facts| + context "on #{os}" do + let(:facts) { os_facts } + + let(:title) { 'test-logs' } + let(:params) do + { + 'paths' => [ + '/var/log/auth.log', + '/var/log/syslog', + ], + 'doc_type' => 'syslog-beat', + } + end + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + end + + it { + is_expected.to contain_file('filebeat-test-logs').with( + notify: 'Service[filebeat]', + ) + } + end + + context "with docker input support on #{os}" do + let(:facts) { os_facts } + + # Docker Support + let(:title) { 'docker' } + let(:params) do + { + 'input_type' => 'docker', + } + end + + if os_facts[:kernel] != 'windows' + it { is_expected.to compile } + + it { + is_expected.to contain_file('filebeat-docker').with( + notify: 'Service[filebeat]', + ) + is_expected.to contain_file('filebeat-docker').with_content( + %r{- type: docker\n\s{2}containers:\n\s{4}ids:\n\s{4}- '\*'\n\s{4}path: /var/lib/docker/containers\n\s{4}stream: all\n\s{2}combine_partial: false\n\s{2}cri.parse_flags: false\n}, + ) + } + end + end + end + + context 'with no parameters' do + let(:title) { 'test-logs' } + let(:params) do + { + 'paths' => [ + '/var/log/auth.log', + '/var/log/syslog', + ], + 'doc_type' => 'syslog-beat', + } + end + + it { is_expected.to raise_error(Puppet::Error) } + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..5e721b7f --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,44 @@ +require 'puppetlabs_spec_helper/module_spec_helper' +require 'rspec-puppet-facts' + +begin + require 'spec_helper_local' if File.file?(File.join(File.dirname(__FILE__), 'spec_helper_local.rb')) +rescue LoadError => loaderror + warn "Could not require spec_helper_local: #{loaderror.message}" +end + +include RspecPuppetFacts + +default_facts = { + puppetversion: Puppet.version, + facterversion: Facter.version, +} + +default_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_facts.yml')) +default_module_facts_path = File.expand_path(File.join(File.dirname(__FILE__), 'default_module_facts.yml')) + +if File.exist?(default_facts_path) && File.readable?(default_facts_path) + default_facts.merge!(YAML.safe_load(File.read(default_facts_path))) +end + +if File.exist?(default_module_facts_path) && File.readable?(default_module_facts_path) + default_facts.merge!(YAML.safe_load(File.read(default_module_facts_path))) +end + +RSpec.configure do |c| + c.default_facts = default_facts + c.before :each do + # set to strictest setting for testing + # by default Puppet runs at warning level + Puppet.settings[:strict] = :warning + end +end + +def ensure_module_defined(module_name) + module_name.split('::').reduce(Object) do |last_module, next_module| + last_module.const_set(next_module, Module.new) unless last_module.const_defined?(next_module) + last_module.const_get(next_module) + end +end + +# 'spec_overrides' from sync.yml will appear below this line diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb new file mode 100644 index 00000000..bfc8d47b --- /dev/null +++ b/spec/spec_helper_acceptance.rb @@ -0,0 +1,38 @@ +require 'beaker-rspec/spec_helper' +require 'beaker-rspec/helpers/serverspec' +require 'beaker/puppet_install_helper' +require 'beaker/module_install_helper' + +run_puppet_install_helper +install_module_on(hosts) +install_module_dependencies_on(hosts) + +UNSUPPORTED_PLATFORMS = ['aix', 'Solaris', 'BSD'].freeze + +RSpec.configure do |c| + module_root = File.expand_path(File.join(File.dirname(__FILE__), '..')) + + c.formatter = :documentation + + # Configure all nodes in nodeset + c.before :suite do + # Install module + puppet_module_install(source: module_root, module_name: 'filebeat') + hosts.each do |host| + on host, puppet('module', 'install', 'puppetlabs-stdlib'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'puppetlabs-apt'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'puppetlabs-powershell'), acceptable_exit_codes: [0, 1] + on host, puppet('module', 'install', 'lwf-remote_file'), acceptable_exit_codes: [0, 1] + end + end +end + +shared_examples 'an idempotent resource' do + it 'apply with no errors' do + apply_manifest(pp, catch_failures: true) + end + + it 'apply a second time without changes', :skip_pup_5016 do + apply_manifest(pp, catch_changes: true) + end +end diff --git a/spec/unit/facter/filebeat_version_spec.rb b/spec/unit/facter/filebeat_version_spec.rb new file mode 100644 index 00000000..f20fdf52 --- /dev/null +++ b/spec/unit/facter/filebeat_version_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe 'filebeat_version' do + before :each do + Facter.clear + Facter.fact(:kernel).stubs(:value).returns('Linux') + end + context 'when on a Linux host' do + before :each do + File.stubs(:executable?) + Facter::Util::Resolution.stubs(:exec) + File.expects(:executable?).with('/usr/share/filebeat/bin/filebeat').returns true + Facter::Util::Resolution.stubs(:exec).with('/usr/share/filebeat/bin/filebeat --version').returns('filebeat version 5.1.1 (amd64), libbeat 5.1.1') + end + it 'returns the correct version' do + expect(Facter.fact(:filebeat_version).value).to eq('5.1.1') + end + end + + context 'when the filebeat package is not installed' do + before :each do + File.stubs(:executable?) + Facter::Util::Resolution.stubs(:exec) + File.expects(:executable?).with('/usr/bin/filebeat').returns false + File.expects(:executable?).with('/usr/local/bin/filebeat').returns false + File.expects(:executable?).with('/usr/share/filebeat/bin/filebeat').returns false + File.expects(:executable?).with('/usr/local/sbin/filebeat').returns false + File.stubs(:exist?) + File.expects(:exist?).with('c:\Program Files\Filebeat\filebeat.exe').returns false + end + it 'returns false' do + expect(Facter.fact(:filebeat_version).value).to eq(false) + end + end +end diff --git a/templates/filebeat.yml.erb b/templates/filebeat.yml.erb new file mode 100644 index 00000000..b063c27d --- /dev/null +++ b/templates/filebeat.yml.erb @@ -0,0 +1,623 @@ +#========================= Filebeat global options ============================ + +<% unless scope.function_versioncmp([@major_version, '6']) >= 0 -%> +filebeat.spool_size: <%= @filebeat_config['filebeat']['spool_size'] %> +filebeat.publish_async: <%= @filebeat_config['filebeat']['publish_async'] %> +filebeat.idle_timeout: <%= @filebeat_config['filebeat']['idle_timeout'] %> +<% end -%> +filebeat.registry_file: <%= @filebeat_config['filebeat']['registry_file'] %> +filebeat.config_dir: <%= @filebeat_config['filebeat']['config_dir'] %> +filebeat.shutdown_timeout: <%= @filebeat_config['filebeat']['shutdown_timeout'] %> + +#================================ General ===================================== + +name: <%= @filebeat_config['beat_name'] %> +<%- if @filebeat_config['tags'].length > 0 -%> +tags: +<%- @filebeat_config['tags'].each do |tag| -%> + - <%= tag %> +<%- end -%> +<%- end -%> + +<%- if @filebeat_config['fields'].length > 0 -%> +fields: + <%- @filebeat_config['fields'].each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> +<%- end -%> + +fields_under_root: <%= @filebeat_config['fields_under_root'] %> +<% unless scope.function_versioncmp([@major_version, '6']) >= 0 -%> +queue_size: <%= @filebeat_config['queue_size'] %> +<% end -%> + +# The internal queue size for bulk events in the processing pipeline. +# Do not modify this value. +#bulk_queue_size: 0 + +<%- if @filebeat_config['max_procs'] != nil -%> +max_procs: <%= @filebeat_config['max_procs'] %> +<%- end -%> + +#================================ Processors ===================================== + +# Processors are used to reduce the number of fields in the exported event or to +# enhance the event with external meta data. This section defines a list of processors +# that are applied one by one and the first one receives the initial event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# Supported processors: drop_fields, drop_event, include_fields +# +# For example, you can use the following processors to keep +# the fields that contain CPU load percentages, but remove the fields that +# contain CPU ticks values: +# +#processors: +#- include_fields: +# fields: ["cpu"] +#- drop_fields: +# fields: ["cpu.user", "cpu.system"] +# +# The following example drops the events that have the HTTP response code 200: +# +#processors: +#- drop_event: +# when: +# equals: +# http.code: 200 +# + +#================================ Outputs ===================================== + +<%- if @filebeat_config['output']['elasticsearch'] != nil -%> +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + <%- if @filebeat_config['output']['elasticsearch']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['elasticsearch']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['elasticsearch']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['loadbalance'] != nil -%> + loadbalance: <%= @filebeat_config['output']['elasticsearch']['loadbalance'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['compression_level'] != nil -%> + compression_level: <%= @filebeat_config['output']['elasticsearch']['compression_level'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['protocol'] != nil -%> + protocol: "<%= @filebeat_config['output']['elasticsearch']['protocol'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['username'] != nil -%> + username: "<%= @filebeat_config['output']['elasticsearch']['username'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['password'] != nil -%> + password: "<%= @filebeat_config['output']['elasticsearch']['password'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['parameters'] != nil && @filebeat_config['output']['elasticsearch']['parameters'].length > 0 -%> + parameters: + <%- @filebeat_config['output']['elasticsearch']['parameters'].each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['elasticsearch']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['index'] != nil -%> + index: "<%= @filebeat_config['output']['elasticsearch']['index'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['pipeline'] != nil -%> + pipeline: "<%= @filebeat_config['output']['elasticsearch']['pipeline'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['pipelines'] != nil -%> + pipelines: + <%- @filebeat_config['output']['elasticsearch']['pipelines'].each do |pipeline| -%> + <%- if pipeline['name'] != nil -%> + - pipeline: "<%= pipeline['name'] %>" + when.equals: + <%= pipeline['filter'] %>: "<%= pipeline['pattern'] %>" + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['indices'] != nil -%> + indices: + <%- @filebeat_config['output']['elasticsearch']['indices'].each do |indice| -%> + <%- if indice['name'] != nil -%> + - index: "<%= indice['name'] %>" + <%- if indice['pipeline'] != nil -%> + pipeline: "<%= indice['pipeline'] %>" + <%- end -%> + when.contains: + <%= indice['filter'] %>: "<%= indice['pattern'] %>" + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['path'] != nil -%> + path: "<%= @filebeat_config['output']['elasticsearch']['path'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['proxy_url'] != nil -%> + proxy_url: "<%= @filebeat_config['output']['elasticsearch']['proxy_url'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['max_retries'] != nil -%> + max_retries: <%= @filebeat_config['output']['elasticsearch']['max_retries'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['elasticsearch']['bulk_max_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['timeout'] != nil -%> + timeout: <%= @filebeat_config['output']['elasticsearch']['timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['flush_interval'] != nil -%> + flush_interval: <%= @filebeat_config['output']['elasticsearch']['flush_interval'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template'] != nil -%> + template: + <%- if @filebeat_config['output']['elasticsearch']['template']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['elasticsearch']['template']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['name'] != nil -%> + name: "<%= @filebeat_config['output']['elasticsearch']['template']['name'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['path'] != nil -%> + path: "<%= @filebeat_config['output']['elasticsearch']['template']['path'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['overwrite'] != nil -%> + overwrite: <%= @filebeat_config['output']['elasticsearch']['template']['name'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x'] != nil -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['enabled'] != nil -%> + versions.2x.enabled: <%= @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['path'] != nil -%> + versions.2x.path: "<%= @filebeat_config['output']['elasticsearch']['template']['versions']['2.x']['path'] %>" + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['elasticsearch']['ssl']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['elasticsearch']['ssl']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['elasticsearch']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['supported_protocols'] != nil -%> + supported_protocols: + <%- @filebeat_config['output']['elasticsearch']['ssl']['supported_protocols'].each do |protocol| -%> + - <%= protocol %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['elasticsearch']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['elasticsearch']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['elasticsearch']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['elasticsearch']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['elasticsearch']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['elasticsearch']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['elasticsearch']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> +<%- if @filebeat_config['output']['logstash'] != nil -%> +#----------------------------- Logstash output -------------------------------- +output.logstash: + <%- if @filebeat_config['output']['logstash']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['logstash']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['logstash']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl_certificate'] != nil -%> + ssl.certificate: <%= @filebeat_config['output']['logstash']['ssl_certificate'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl_key'] != nil -%> + ssl.key: <%= @filebeat_config['output']['logstash']['ssl_key'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['logstash']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['compression_level'] != nil -%> + compression_level: <%= @filebeat_config['output']['logstash']['compression_level'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['loadbalance'] != nil -%> + loadbalance: <%= @filebeat_config['output']['logstash']['loadbalance'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['pipelining'] != nil -%> + pipelining: <%= @filebeat_config['output']['logstash']['pipelining'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['index'] != nil -%> + index: <%= @filebeat_config['output']['logstash']['index'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['proxy_url'] != nil -%> + proxy_url: <%= @filebeat_config['output']['logstash']['proxy_url'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['proxy_use_local_resolver'] != nil -%> + proxy_use_local_resolver: <%= @filebeat_config['output']['logstash']['proxy_use_local_resolver'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['logstash']['ssl']['enabled'] == false -%> + enabled: false + <%- else -%> + enabled: true + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['logstash']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['logstash']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['logstash']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['logstash']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['logstash']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['logstash']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['logstash']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['logstash']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> +<%- if @filebeat_config['output']['kafka'] != nil -%> +#------------------------------- Kafka output --------------------------------- +output.kafka: + <%- if @filebeat_config['output']['kafka']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['kafka']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['kafka']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['topic'] != nil -%> + topic: <%= @filebeat_config['output']['kafka']['topic'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['round_robin'] != nil -%> + round_robin: + group_events: <%= @filebeat_config['output']['kafka']['round_robin']['group_events'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['key'] != nil -%> + key: '<%= @filebeat_config['output']['kafka']['key'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['codec.format'] != nil -%> + codec.format: + <%- if @filebeat_config['output']['kafka']['codec.format']['string'] != nil -%> + string: <%= @filebeat_config['output']['kafka']['codec.format']['string'] %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['partition'] != nil and @filebeat_config['output']['kafka']['partition']['hash'] != nil -%> + partition.hash: + <%- if @filebeat_config['output']['kafka']['partition']['hash']['reachable_only'] != nil -%> + reachable_only: <%= @filebeat_config['output']['kafka']['partition']['hash']['reachable_only'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['partition']['hash']['random'] != nil -%> + random: <%= @filebeat_config['output']['kafka']['partition']['hash']['random'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['partition']['hash']['hash'] != nil -%> + hash: + <%- @filebeat_config['output']['kafka']['partition']['hash']['hash'].each do |value| -%> + - <%= value %> + <%- end -%> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['username'] != nil -%> + username: '<%= @filebeat_config['output']['kafka']['username'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['password'] != nil -%> + password: '<%= @filebeat_config['output']['kafka']['password'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['version'] != nil -%> + version: <%= @filebeat_config['output']['kafka']['version'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['metadata'] != nil -%> + metadata: + <%- if @filebeat_config['output']['kafka']['retry']['max'] != nil -%> + retry.max: 3 + <%- end -%> + <%- if @filebeat_config['output']['kafka']['retry']['backoff'] != nil -%> + retry.backoff: 3 + <%- end -%> + <%- if @filebeat_config['output']['kafka']['retry']['frequency'] != nil -%> + retry.frequency: 3 + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['kafka']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['max_retries'] != nil -%> + max_retries: <%= @filebeat_config['output']['kafka']['max_retries'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['kafka']['bulk_max_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['timeout'] != nil -%> + timeout: <%= @filebeat_config['output']['kafka']['timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['broker_timeout'] != nil -%> + broker_timeout: <%= @filebeat_config['output']['kafka']['broker_timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['channel_buffer_size'] != nil -%> + channel_buffer_size: <%= @filebeat_config['output']['kafka']['channel_buffer_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['keep_alive'] != nil -%> + keep_alive: <%= @filebeat_config['output']['kafka']['keep_alive'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['compression'] != nil -%> + compression: <%= @filebeat_config['output']['kafka']['compression'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['max_message_bytes'] != nil -%> + max_message_bytes: <%= @filebeat_config['output']['kafka']['max_message_bytes'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['required_acks'] != nil -%> + required_acks: <%= @filebeat_config['output']['kafka']['required_acks'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['flush_interval'] != nil -%> + flush_interval: <%= @filebeat_config['output']['kafka']['flush_interval'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['client_id'] != nil -%> + client_id: <%= @filebeat_config['output']['kafka']['client_id'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['kafka']['ssl']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['kafka']['ssl']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['kafka']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['supported_protocols'] != nil -%> + supported_protocols: + <%- @filebeat_config['output']['kafka']['ssl']['supported_protocols'].each do |protocol| -%> + - <%= protocol %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['kafka']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['kafka']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['kafka']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['kafka']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['kafka']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['kafka']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['kafka']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> +<%- if @filebeat_config['output']['redis'] != nil -%> +#------------------------------- Redis output --------------------------------- +output.redis: + <%- if @filebeat_config['output']['redis']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['redis']['enabled'] %> + <%- end -%> + hosts: + <%- @filebeat_config['output']['redis']['hosts'].each do |host| -%> + - <%= host %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['port'] != nil -%> + port: <%= @filebeat_config['output']['redis']['port'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['key'] != nil -%> + key: <%= @filebeat_config['output']['redis']['key'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['password'] != nil -%> + password: <%= @filebeat_config['output']['redis']['password'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['db'] != nil -%> + db: <%= @filebeat_config['output']['redis']['db'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['datetype'] != nil -%> + datetype: <%= @filebeat_config['output']['redis']['datetype'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['worker'] != nil -%> + worker: <%= @filebeat_config['output']['redis']['worker'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['loadbalance'] != nil -%> + loadbalance: <%= @filebeat_config['output']['redis']['loadbalance'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['timeout'] != nil -%> + timeout: <%= @filebeat_config['output']['redis']['timeout'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['max_retries'] != nil -%> + max_retries: <%= @filebeat_config['output']['redis']['max_retries'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['redis']['bulk_max_size'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['proxy_url'] != nil -%> + proxy_url: <%= @filebeat_config['output']['redis']['proxy_url'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['proxy_use_local_resolver'] != nil -%> + proxy_use_local_resolver: <%= @filebeat_config['output']['redis']['proxy_use_local_resolver'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl'] != nil -%> + ssl: + <%- if @filebeat_config['output']['redis']['ssl']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['logstash']['ssl']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['verification_mode'] != nil -%> + verification_mode: <%= @filebeat_config['output']['redis']['ssl']['verification_mode'] %> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['supported_protocols'] != nil -%> + supported_protocols: + <%- @filebeat_config['output']['redis']['ssl']['supported_protocols'].each do |protocol| -%> + - <%= protocol %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['certificate_authorities'] != nil -%> + certificate_authorities: + <%- @filebeat_config['output']['redis']['ssl']['certificate_authorities'].each do |ca| -%> + - <%= ca %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['certificate'] != nil -%> + certificate: "<%= @filebeat_config['output']['redis']['ssl']['certificate'] %>" + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['key'] != nil -%> + key: "<%= @filebeat_config['output']['redis']['ssl']['key'] %>" + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['key_passphrase'] != nil -%> + key_passphrase: '<%= @filebeat_config['output']['redis']['ssl']['key_passphrase'] %>' + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['cipher_suites'] != nil -%> + cipher_suites: + <%- @filebeat_config['output']['redis']['ssl']['cipher_suites'].each do |cipher_suite| -%> + - <%= cipher_suite %> + <%- end -%> + <%- end -%> + <%- if @filebeat_config['output']['redis']['ssl']['curve_types'] != nil -%> + curve_types: + <%- @filebeat_config['output']['redis']['ssl']['curve_types'].each do |curve_type| -%> + - <%= curve_type %> + <%- end -%> + <%- end -%> + <%- end -%> +<%- end -%> +<%- if @filebeat_config['output']['file'] != nil -%> +#------------------------------- File output ---------------------------------- +output.file: + <%- if @filebeat_config['output']['file']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['file']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['file']['path'] != nil -%> + path: "<%= @filebeat_config['output']['file']['path'] %>" + <%- end -%> + <%- if @filebeat_config['output']['file']['filename'] != nil -%> + filename: <%= @filebeat_config['output']['file']['filename'] %> + <%- end -%> + <%- if @filebeat_config['output']['file']['rotate_every_kb'] != nil -%> + rotate_every_kb: <%= @filebeat_config['output']['file']['rotate_every_kb'] %> + <%- end -%> + <%- if @filebeat_config['output']['file']['number_of_files'] != nil -%> + number_of_files: <%= @filebeat_config['output']['file']['number_of_files'] %> + <%- end -%> +<%- end -%> +<%- if @filebeat_config['output']['console'] != nil -%> +#-------------------------- Console output ------------------------------------ +output.console: + <%- if @filebeat_config['output']['console']['enabled'] != nil -%> + enabled: <%= @filebeat_config['output']['console']['enabled'] %> + <%- end -%> + <%- if @filebeat_config['output']['console']['pretty'] != nil -%> + pretty: <%= @filebeat_config['output']['console']['pretty'] %> + <%- end -%> + <%- if @filebeat_config['output']['console']['bulk_max_size'] != nil -%> + bulk_max_size: <%= @filebeat_config['output']['console']['bulk_max_size'] %> + <%- end -%> +<%- end -%> + +#================================= Paths ====================================== + +# The home path for the filebeat installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the filebeat installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: <%= @filebeat_config['filebeat']['config_dir'] %> + +# The data path for the filebeat installation. This is the default base path +# for all the files in which filebeat needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data + +# The logs path for a filebeat installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs + +#================================ Logging ===================================== +<%- if @filebeat_config['logging']['level'] != nil -%> +logging.level: <%= @filebeat_config['logging']['level'] %> +<%- end -%> +<%- if @filebeat_config['logging']['selectors'] != nil && @filebeat_config['logging']['selectors'].length > 0 -%> +logging.selectors: +<%- @filebeat_config['logging']['selectors'].each do |selector| -%> + - <%= selector %> +<%- end -%> +<%- end -%> +<%- if @filebeat_config['logging']['to_syslog'] != nil -%> +logging.to_syslog: <%= @filebeat_config['logging']['to_syslog'] %> +<%- end -%> +<%- if @filebeat_config['logging']['metrics'] != nil -%> +<%- if @filebeat_config['logging']['metrics']['enabled'] != nil -%> +logging.metrics.enabled: <%= @filebeat_config['logging']['metrics']['enabled'] %> +<%- end -%> +<%- if @filebeat_config['logging']['metrics']['period'] != nil -%> +logging.metrics.period: <%= @filebeat_config['logging']['metrics']['period'] %> +<%- end -%> +<%- end -%> +<%- if @filebeat_config['logging']['to_files'] != nil -%> +logging.to_files: <%= @filebeat_config['logging']['to_files'] %> +<%- end -%> +<%- if @filebeat_config['logging']['files'] != nil -%> +logging.files: + <%- if @filebeat_config['logging']['files']['path'] != nil -%> + path: <%= @filebeat_config['logging']['files']['path'] %> + <%- end -%> + <%- if @filebeat_config['logging']['files']['name'] != nil -%> + name: <%= @filebeat_config['logging']['files']['name'] %> + <%- end -%> + <%- if @filebeat_config['logging']['files']['rotateeverybytes'] != nil -%> + rotateeverybytes: <%= @filebeat_config['logging']['files']['rotateeverybytes'] %> + <%- end -%> + <%- if @filebeat_config['logging']['files']['keepfiles'] != nil -%> + keepfiles: <%= @filebeat_config['logging']['files']['keepfiles'] %> + <%- end -%> +<%- end -%> diff --git a/templates/prospector.yml.erb b/templates/prospector.yml.erb new file mode 100644 index 00000000..1f3416a6 --- /dev/null +++ b/templates/prospector.yml.erb @@ -0,0 +1,198 @@ +--- +- type: <%= @input_type %> + <%- if @input_type != 'docker' -%> + paths: + <%- @paths.each do |log_path| -%> + - <%= log_path %> + <%- end -%> + <%- else -%> + containers: + ids: + <%- @containers_ids.each do |id| -%> + - <%= id %> + <%- end -%> + path: <%= @containers_path %> + stream: <%= @containers_stream %> + combine_partial: <%= @combine_partial %> + cri.parse_flags: <%= @cri_parse_flags %> + <%- end %> + <%- if @encoding -%> + encoding: <%= @encoding %> + <%- end -%> + <%- if @include_lines.length > 0 -%> + include_lines: + <%- @include_lines.each do |include_line| -%> + - '<%= include_line %>' + <%- end -%> + <%- end -%> + <%- if @exclude_lines.length > 0 -%> + exclude_lines: + <%- @exclude_lines.each do |exclude_line| -%> + - '<%= exclude_line %>' + <%- end -%> + <%- end -%> + <%- if @exclude_files.length > 0 -%> + exclude_files: + <%- @exclude_files.each do |exclude_file| -%> + - <%= exclude_file %> + <%- end -%> + <%- end -%> + <%- if @fields.length > 0 -%> + fields: + <%- @fields.each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + fields_under_root: <%= @fields_under_root %> + <%- if @tags.length > 0 -%> + tags: + <%- @tags.each do |tag| -%> + - <%= tag %> + <%- end -%> + <%- end -%> + <%- if @ignore_older -%> + ignore_older: <%= @ignore_older %> + <%- end -%> + <%- if @doc_type -%> + document_type: <%= @doc_type %> + <%- end -%> + <%- if @scan_frequency -%> + scan_frequency: <%= @scan_frequency %> + <%- end -%> + <%- if @harvester_buffer_size -%> + harvester_buffer_size: <%= @harvester_buffer_size %> + <%- end -%> + <%- if @max_bytes -%> + max_bytes: <%= @max_bytes %> + <%- end -%> + <%- if @symlinks -%> + symlinks: <%= @symlinks %> + <%- end -%> + <%- if @close_older -%> + close_older: <%= @close_older %> + <%- end -%> + <%- if @force_close_files -%> + force_close_files: <%= @force_close_files %> + <%- end -%> + <%- if @pipeline -%> + pipeline: <%= @pipeline %> + <%- end -%> + + <%- if @json.length > 0 -%> + ### JSON configuration + json: + # Decode JSON options. Enable this if your logs are structured in JSON. + # JSON key on which to apply the line filtering and multiline settings. This key + # must be top level and its value must be string, otherwise it is ignored. If + # no text key is defined, the line filtering and multiline features cannot be used. + <%- if @json['message_key'] != nil-%> + message_key: '<%= @json['message_key'] %>' + <%- end -%> + + # By default, the decoded JSON is placed under a "json" key in the output document. + # If you enable this setting, the keys are copied top level in the output document. + <%- if @json['keys_under_root'] != nil -%> + keys_under_root: <%= @json['keys_under_root'] %> + <%- end -%> + + # If keys_under_root and this setting are enabled, then the values from the decoded + # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) + # in case of conflicts. + <%- if @json['overwrite_keys'] != nil -%> + overwrite_keys: <%= @json['overwrite_keys'] %> + <%- end -%> + + # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON + # unmarshaling errors or when a text key is defined in the configuration but cannot + # be used. + <%- if @json['add_error_key'] != nil -%> + add_error_key: <%= @json['add_error_key'] %> + <%- end -%> + <%- end -%> + + <%- if @multiline.length > 0 -%> + multiline: + <%- if @multiline['pattern'] -%> + pattern: '<%= @multiline['pattern'] %>' + <%- end -%> + <%- if @multiline['negate'] -%> + negate: <%= @multiline['negate'] %> + <%- end -%> + <%- if @multiline['match'] -%> + match: <%= @multiline['match'] %> + <%- end -%> + <%- if @multiline['max_lines'] -%> + max_lines: <%= @multiline['max_lines'] %> + <%- end -%> + <%- if @multiline['timeout'] -%> + timeout: <%= @multiline['timeout'] %> + <%- end -%> + <%- end -%> + tail_files: <%= @tail_files %> + + # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the + # original for harvesting but will report the symlink name as source. + #symlinks: false + + <%- if @backoff -%> + backoff: <%= @backoff %> + <%- end -%> + <%- if @max_backoff -%> + max_backoff: <%= @max_backoff %> + <%- end -%> + <%- if @backoff_factor -%> + backoff_factor: <%= @backoff_factor %> + <%- end -%> + + # Experimental: Max number of harvesters that are started in parallel. + # Default is 0 which means unlimited + <%- if @harvester_limit -%> + harvester_limit: <%= @harvester_limit %> + <%- end -%> + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + <%- if @close_inactive -%> + close_inactive: <%= @close_inactive %> + <%- end -%> + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_renamed: <%= @close_renamed %> + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + close_removed: <%= @close_removed %> + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_eof: <%= @close_eof %> + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + <%- if @clean_inactive -%> + clean_inactive: <%= @clean_inactive %> + <%- end -%> + + # Removes the state for file which cannot be found on disk anymore immediately + clean_removed: <%= @clean_removed %> + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + <%- if @close_timeout -%> + close_timeout: <%= @close_timeout %> + <%- end -%> + <%- if @processors.length > 0 -%> + # Managing processors releated only for specified prospector + processors: + <%- %><%= @processors.to_yaml.lines.drop(1).join.gsub(/^/, ' ') -%> + <%- end -%> diff --git a/templates/prospector5.yml.erb b/templates/prospector5.yml.erb new file mode 100644 index 00000000..bd11358a --- /dev/null +++ b/templates/prospector5.yml.erb @@ -0,0 +1,183 @@ +--- +filebeat: + prospectors: + - <% if scope.function_versioncmp([scope.lookupvar('filebeat::major_version'), '6']) >= 0 %>type<% else %>input_type<% end %>: <%= @input_type %> + paths: + <%- @paths.each do |log_path| -%> + - <%= log_path %> + <%- end -%> + <%- if @encoding -%> + encoding: <%= @encoding %> + <%- end -%> + <%- if @include_lines.length > 0 -%> + include_lines: + <%- @include_lines.each do |include_line| -%> + - '<%= include_line %>' + <%- end -%> + <%- end -%> + <%- if @exclude_lines.length > 0 -%> + exclude_lines: + <%- @exclude_lines.each do |exclude_line| -%> + - '<%= exclude_line %>' + <%- end -%> + <%- end -%> + <%- if @exclude_files.length > 0 -%> + exclude_files: + <%- @exclude_files.each do |exclude_file| -%> + - <%= exclude_file %> + <%- end -%> + <%- end -%> + <%- if @fields.length > 0 -%> + fields: + <%- @fields.each_pair do |k, v| -%> + <%= k %>: <%= v %> + <%- end -%> + <%- end -%> + fields_under_root: <%= @fields_under_root %> + <%- if @tags.length > 0 -%> + tags: + <%- @tags.each do |tag| -%> + - <%= tag %> + <%- end -%> + <%- end -%> + <%- if @ignore_older -%> + ignore_older: <%= @ignore_older %> + <%- end -%> + <%- if @doc_type -%> + document_type: <%= @doc_type %> + <%- end -%> + <%- if @scan_frequency -%> + scan_frequency: <%= @scan_frequency %> + <%- end -%> + <%- if @harvester_buffer_size -%> + harvester_buffer_size: <%= @harvester_buffer_size %> + <%- end -%> + <%- if @max_bytes -%> + max_bytes: <%= @max_bytes %> + <%- end -%> + <%- if @symlinks -%> + symlinks: <%= @symlinks %> + <%- end -%> + <%- if @close_older -%> + close_older: <%= @close_older %> + <%- end -%> + <%- if @force_close_files -%> + force_close_files: <%= @force_close_files %> + <%- end -%> + <%- if @pipeline -%> + pipeline: <%= @pipeline %> + <%- end -%> + + <%- if @json.length > 0 -%> + ### JSON configuration + json: + # Decode JSON options. Enable this if your logs are structured in JSON. + # JSON key on which to apply the line filtering and multiline settings. This key + # must be top level and its value must be string, otherwise it is ignored. If + # no text key is defined, the line filtering and multiline features cannot be used. + <%- if @json['message_key'] != nil-%> + message_key: '<%= @json['message_key'] %>' + <%- end -%> + + # By default, the decoded JSON is placed under a "json" key in the output document. + # If you enable this setting, the keys are copied top level in the output document. + <%- if @json['keys_under_root'] != nil -%> + keys_under_root: <%= @json['keys_under_root'] %> + <%- end -%> + + # If keys_under_root and this setting are enabled, then the values from the decoded + # JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) + # in case of conflicts. + <%- if @json['overwrite_keys'] != nil -%> + overwrite_keys: <%= @json['overwrite_keys'] %> + <%- end -%> + + # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON + # unmarshaling errors or when a text key is defined in the configuration but cannot + # be used. + <%- if @json['add_error_key'] != nil -%> + add_error_key: <%= @json['add_error_key'] %> + <%- end -%> + <%- end -%> + + <%- if @multiline.length > 0 -%> + multiline: + <%- if @multiline['pattern'] -%> + pattern: '<%= @multiline['pattern'] %>' + <%- end -%> + <%- if @multiline['negate'] -%> + negate: <%= @multiline['negate'] %> + <%- end -%> + <%- if @multiline['match'] -%> + match: <%= @multiline['match'] %> + <%- end -%> + <%- if @multiline['max_lines'] -%> + max_lines: <%= @multiline['max_lines'] %> + <%- end -%> + <%- if @multiline['timeout'] -%> + timeout: <%= @multiline['timeout'] %> + <%- end -%> + <%- end -%> + tail_files: <%= @tail_files %> + + # Experimental: If symlinks is enabled, symlinks are opened and harvested. The harvester is openening the + # original for harvesting but will report the symlink name as source. + #symlinks: false + + <%- if @backoff -%> + backoff: <%= @backoff %> + <%- end -%> + <%- if @max_backoff -%> + max_backoff: <%= @max_backoff %> + <%- end -%> + <%- if @backoff_factor -%> + backoff_factor: <%= @backoff_factor %> + <%- end -%> + + # Experimental: Max number of harvesters that are started in parallel. + # Default is 0 which means unlimited + <%- if @harvester_limit -%> + harvester_limit: <%= @harvester_limit %> + <%- end -%> + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + <%- if @close_inactive -%> + close_inactive: <%= @close_inactive %> + <%- end -%> + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_renamed: <%= @close_renamed %> + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + close_removed: <%= @close_removed %> + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + close_eof: <%= @close_eof %> + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + <%- if @clean_inactive -%> + clean_inactive: <%= @clean_inactive %> + <%- end -%> + + # Removes the state for file which cannot be found on disk anymore immediately + clean_removed: <%= @clean_removed %> + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + <%- if @close_timeout -%> + close_timeout: <%= @close_timeout %> + <%- end -%> diff --git a/templates/pure_hash.yml.erb b/templates/pure_hash.yml.erb new file mode 100644 index 00000000..04e16804 --- /dev/null +++ b/templates/pure_hash.yml.erb @@ -0,0 +1,2 @@ +### Filebeat configuration managed by Puppet ### +<%= @filebeat_config.to_yaml() %>