diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ddd526..dc833cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changes +## 3.0 / 2019-12-18 + +**This role will install Agent v7 by default.** Datadog Agent v7 runs checks with Python 3, so if you were running any custom checks written in Python, they must be compatible with Python 3. If you were not running any custom checks or if your custom checks are already compatible with Python 3, then it is safe to upgrade to Agent v7. + +This release contains breaking changes to the pillar file structure. Please read the `README` file, in particular the `Pillar configuration`, as well as the [pillar example file](pillar.example) for details on how to write your pillar file. + +* [FEATURE] Add Agent v7 support [#40][] +* [FEATURE] Allow all `datadog.yaml` config options for Agent v6 and v7 [#35][] +* [FEATURE] Add ability to choose check versions [#37][] +* [BUGFIX] Put check config files in the `.d/` directory on Agent v6 and v7 [#39][] +* [BUGFIX] Fix Agent v6 and v7 install from beta repositories [#42][] +* [BUGFIX] Fix checks config directory permissions [#45][] +* [OTHER] Testing docs & scripts improvements [#44] +* [OTHER] Do not import the old rpm GPG key if installing Agent v7 [#43][] +* [OTHER] Unused variable cleanup [#36][] + ## 2.1 / 2019-08-27 * [FEATURE] Add python_version option [#33][] @@ -40,3 +56,11 @@ [#31]: https://github.com/DataDog/datadog-formula/issues/31 [#32]: https://github.com/DataDog/datadog-formula/issues/32 [#33]: https://github.com/DataDog/datadog-formula/issues/33 +[#35]: https://github.com/DataDog/datadog-formula/issues/35 +[#36]: https://github.com/DataDog/datadog-formula/issues/36 +[#37]: https://github.com/DataDog/datadog-formula/issues/37 +[#39]: https://github.com/DataDog/datadog-formula/issues/39 +[#40]: https://github.com/DataDog/datadog-formula/issues/40 +[#42]: https://github.com/DataDog/datadog-formula/issues/42 +[#43]: https://github.com/DataDog/datadog-formula/issues/43 +[#45]: https://github.com/DataDog/datadog-formula/issues/45 \ No newline at end of file diff --git a/README.rst b/README.rst index 72e3b4b..8ecb1a8 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,72 @@ Agent itself and the checks. ``datadog.uninstall`` ------------------ -Stops the service and uninstall Datadog Agent. +Stops the service and uninstalls Datadog Agent. + +Pillar configuration +==================== + +The formula configuration must be written in the ``datadog`` key of the pillar file. + +The formula configuration contains three parts: ``config``, ``install_settings``, and ``checks``. + +``config`` +---------- +The ``config`` option contains the configuration options which will be written in the minions' Agent configuration file (``datadog.yaml`` for Agent v6 & v7, ``datadog.conf`` for Agent v5). + +Depending on the Agent version installed, different options can be set: + +- Agent v6 & v7: all options supported by the Agent's configuration file are supported. +- Agent v5: only the ``api_key`` option is supported. + +Example: set the API key, and the site option to ``datadoghq.eu`` (Agent v6 only) + +.. code:: + + datadog: + config: + api_key: + site: datadoghq.eu + +``install_settings`` +-------------------- +The ``install_settings`` option contains the Agent installation configuration options. +It has the following option: + +- ``agent_version``: the version of the Agent which will be installed. Default: latest (will install the latest Agent v7 package). + +Example: install the Agent version ``6.14.1`` + +.. code:: + + datadog: + install_settings: + agent_version: 6.14.1 + + +``checks`` +---------- +The ``checks`` option contains configuration for the Agent Checks. + +To add an Agent Check, add an entry in the ``checks`` option with the check's name as the key. + +Each check has two options: + +- ``config``: contains the check's configuration, which will be written to the check's configuration file (``/.d/conf.yaml`` for Agent v6/v7, ``/.yaml`` for Agent v5). +- ``version``: the version of the check which will be installed (Agent v6 and v7 only). Default: the version bundled with the agent. + +Example: ``directory`` check version ``1.4.0``, monitoring the ``/srv/pillar`` directory + +.. code:: + + datadog: + checks: + directory: + config: + instances: + - directory: "/srv/pillar" + name: "pillars" + version: 1.4.0 Development =========== @@ -82,6 +147,6 @@ Run the formula .. code-block:: shell $ cd test/ - $ TEST_DIST=debian docker-compose up + $ TEST_DIST=debian docker-compose up --build You should be able to see from the logs if all the states completed successfully. diff --git a/datadog/config.sls b/datadog/config.sls index e53969c..f95a25b 100644 --- a/datadog/config.sls +++ b/datadog/config.sls @@ -1,61 +1,49 @@ -{% from "datadog/map.jinja" import datadog_settings with context %} -{% set config_file_path = '%s/%s'|format(datadog_settings.config_folder, datadog_settings.config_file) -%} -{% set example_file_path = '%s.example'|format(config_file_path) -%} +{% from "datadog/map.jinja" import datadog_config, datadog_install_settings, datadog_checks, latest_agent_version, parsed_version with context %} +{% set config_file_path = '%s/%s'|format(datadog_install_settings.config_folder, datadog_install_settings.config_file) -%} -datadog-example: - file.copy: +{%- if not latest_agent_version and parsed_version[1] == '5' %} +datadog_conf_installed: + file.managed: - name: {{ config_file_path }} - - source: {{ example_file_path }} - # file.copy will not overwrite a named file, so we only need to check if the example config file exists - - onlyif: test -f {{ example_file_path }} + - source: salt://datadog/files/datadog.conf.jinja + - user: dd-agent + - group: dd-agent + - mode: 600 + - template: jinja - require: - pkg: datadog-pkg - -{% if datadog_settings.api_key is defined %} -datadog-conf-api-key: - file.replace: +{%- else %} +datadog_yaml_installed: + file.managed: - name: {{ config_file_path }} - - pattern: "api_key:(.*)" - - repl: "api_key: {{ datadog_settings.api_key }}" - - count: 1 - - onlyif: test -f {{ config_file_path }} - - watch: + - source: salt://datadog/files/datadog.yaml.jinja + - user: dd-agent + - group: dd-agent + - mode: 600 + - template: jinja + - require: - pkg: datadog-pkg -{% endif %} +{%- endif %} -datadog-conf-site: - file.replace: - - name: {{ config_file_path }} - - pattern: "(.*)site:(.*)" -{% if datadog_settings.site is defined %} - - repl: "site: {{ datadog_settings.site }}" -{% else %} - - repl: "# site: datadoghq.com" -{% endif %} - - count: 1 - - onlyif: test -f {{ config_file_path }} - - watch: - - pkg: datadog-pkg +{% if datadog_checks is defined %} +{% for check_name in datadog_checks %} -datadog-conf-python-version: - file.replace: - - name: {{ config_file_path }} - - pattern: "(.*)python_version:(.*)" -{% if datadog_settings.python_version is defined %} - - repl: "python_version: {{ datadog_settings.python_version }}" -{% else %} - - repl: "# python_version: 2" -{% endif %} - - count: 1 - - onlyif: test -f {{ config_file_path }} - - watch: - - pkg: datadog-pkg +{%- if latest_agent_version or parsed_version[1] != '5' %} +datadog_{{ check_name }}_folder_installed: + file.directory: + - name: {{ datadog_install_settings.confd_path }}/{{ check_name }}.d + - user: dd-agent + - group: root + - mode: 700 +{%- endif %} -{% if datadog_settings.checks is defined %} -{% for check_name in datadog_settings.checks %} datadog_{{ check_name }}_yaml_installed: file.managed: - - name: {{ datadog_settings.checks_confd }}/{{ check_name }}.yaml + {%- if latest_agent_version or parsed_version[1] != '5' %} + - name: {{ datadog_install_settings.confd_path }}/{{ check_name }}.d/conf.yaml + {%- else %} + - name: {{ datadog_install_settings.confd_path }}/{{ check_name }}.yaml + {%- endif %} - source: salt://datadog/files/conf.yaml.jinja - user: dd-agent - group: root @@ -63,5 +51,14 @@ datadog_{{ check_name }}_yaml_installed: - template: jinja - context: check_name: {{ check_name }} + +{%- if latest_agent_version or parsed_version[1] != '5' %} +{%- if datadog_checks[check_name].version is defined %} +datadog_{{ check_name }}_version_{{ datadog_checks[check_name].version }}_installed: + cmd.run: + - name: sudo -u dd-agent datadog-agent integration install datadog-{{ check_name }}=={{ datadog_checks[check_name].version }} +{%- endif %} +{%- endif %} + {% endfor %} {% endif %} diff --git a/datadog/files/conf.yaml.jinja b/datadog/files/conf.yaml.jinja index 4a02275..31e2e60 100644 --- a/datadog/files/conf.yaml.jinja +++ b/datadog/files/conf.yaml.jinja @@ -1,5 +1,11 @@ -{% if pillar.datadog.checks[check_name].init_config is not defined -%} +{% from "datadog/map.jinja" import datadog_checks with context -%} + +{% if datadog_checks[check_name].config is defined -%} + +{% if datadog_checks[check_name].config.init_config is not defined -%} init_config: {% endif -%} -{{ pillar.datadog.checks[check_name] | yaml(False) }} +{{ datadog_checks[check_name].config | yaml(False) }} + +{% endif -%} diff --git a/datadog/files/datadog.conf.jinja b/datadog/files/datadog.conf.jinja new file mode 100644 index 0000000..c282f84 --- /dev/null +++ b/datadog/files/datadog.conf.jinja @@ -0,0 +1,10 @@ +{% from "datadog/map.jinja" import datadog_config with context -%} + +[Main] +dd_url: https://app.datadoghq.com + +{% if datadog_config.api_key is not defined -%} +api_key: +{% else -%} +api_key: {{ datadog_config.api_key }} +{% endif -%} diff --git a/datadog/files/datadog.yaml.jinja b/datadog/files/datadog.yaml.jinja new file mode 100644 index 0000000..5ccfa56 --- /dev/null +++ b/datadog/files/datadog.yaml.jinja @@ -0,0 +1,9 @@ +{% from "datadog/map.jinja" import datadog_config with context -%} + +{% if datadog_config.api_key is not defined -%} +api_key: +{% endif -%} + +{% if datadog_config | length -%} +{{ datadog_config | yaml(False) }} +{% endif -%} diff --git a/datadog/init.sls b/datadog/init.sls index da93541..b524ce2 100644 --- a/datadog/init.sls +++ b/datadog/init.sls @@ -1,4 +1,4 @@ include: - - datadog.config - datadog.install + - datadog.config - datadog.service diff --git a/datadog/install.sls b/datadog/install.sls index 3bb1941..5ac42f0 100644 --- a/datadog/install.sls +++ b/datadog/install.sls @@ -1,4 +1,4 @@ -{% from "datadog/map.jinja" import datadog_settings, latest_agent_version, parsed_version with context %} +{% from "datadog/map.jinja" import datadog_install_settings, latest_agent_version, parsed_version with context %} {%- if grains['os_family'].lower() == 'debian' %} datadog-apt-https: @@ -17,7 +17,9 @@ datadog-repo: {% set distribution = 'stable' %} {%- endif %} {#- Determine which channel we should look in #} - {%- if latest_agent_version or parsed_version[1] == '6' %} + {%- if latest_agent_version or parsed_version[1] == '7' %} + {% set packages = '7' %} + {%- elif parsed_version[1] == '6' %} {% set packages = '6' %} {%- else %} {% set packages = 'main' %} @@ -34,8 +36,16 @@ datadog-repo: {%- elif grains['os_family'].lower() == 'redhat' %} {#- Determine the location of the package we want #} {%- if not latest_agent_version and (parsed_version[2] == 'beta' or parsed_version[2] == 'rc') %} - {% set path = 'beta' %} - {%- elif latest_agent_version or parsed_version[1] == '6' %} + {%- if parsed_version[1] == '7' %} + {% set path = 'beta/7' %} + {%- elif parsed_version[1] == '6' %} + {% set path = 'beta/6' %} + {%- else %} + {% set path = 'beta' %} + {%- endif %} + {%- elif latest_agent_version or parsed_version[1] == '7' %} + {% set path = 'stable/7' %} + {%- elif parsed_version[1] == '6' %} {% set path = 'stable/6' %} {%- else %} {% set path = 'rpm' %} @@ -43,21 +53,25 @@ datadog-repo: - name: datadog - baseurl: https://yum.datadoghq.com/{{ path }}/{{ grains['cpuarch'] }} - gpgcheck: '1' + {%- if latest_agent_version or parsed_version[1] == '7' %} + - gpgkey: https://yum.datadoghq.com/DATADOG_RPM_KEY_E09422B3.public + {%- else %} - gpgkey: https://yum.datadoghq.com/DATADOG_RPM_KEY_E09422B3.public https://yum.datadoghq.com/DATADOG_RPM_KEY.public + {%- endif %} - sslverify: '1' {% endif %} datadog-pkg: pkg.installed: - - name: {{ datadog_settings.pkg_name }} + - name: datadog-agent {%- if latest_agent_version %} - version: 'latest' {%- elif grains['os_family'].lower() == 'debian' %} - - version: 1:{{ datadog_settings.agent_version }}-1 + - version: 1:{{ datadog_install_settings.agent_version }}-1 {%- elif grains['os_family'].lower() == 'redhat' %} - - version: {{ datadog_settings.agent_version }}-1 + - version: {{ datadog_install_settings.agent_version }}-1 {%- endif %} - ignore_epoch: True - refresh: True - require: - - pkgrepo: datadog-repo + - pkgrepo: datadog-repo \ No newline at end of file diff --git a/datadog/map.jinja b/datadog/map.jinja index 0e7f75f..091ad7d 100644 --- a/datadog/map.jinja +++ b/datadog/map.jinja @@ -7,48 +7,50 @@ {% set default_settings = { 'datadog': { - 'pkg_name': 'datadog-agent', - 'service_name': 'datadog-agent', - 'api_key': 'aaaaaaaabbbbbbbbccccccccdddddddd', - 'agent_version': 'latest', + 'config': {}, + 'checks': {}, + 'install_settings': { + 'agent_version': 'latest', + }, } }%} {# Merge os_family_map into the default settings #} -{% do default_settings.datadog.update(os_family_map) %} +{% do default_settings.datadog.update(os_family_map) %} {# Merge in datadog pillar #} -{% set datadog_settings = salt['pillar.get']('datadog', default=default_settings.datadog, merge=True) %} +{% set datadog = salt['pillar.get']('datadog', default=default_settings.datadog, merge=True) %} +{% set datadog_config = datadog['config'] %} +{% set datadog_checks = datadog['checks'] %} +{% set datadog_install_settings = datadog['install_settings'] %} {# Determine if we're looking for the latest package or a specific version #} -{%- if datadog_settings.agent_version == 'latest' %} +{%- if datadog_install_settings.agent_version == 'latest' %} {%- set latest_agent_version = true %} {%- else %} {%- set latest_agent_version = false %} - {%- set parsed_version = datadog_settings.agent_version | regex_match('(([0-9]+)\.[0-9]+\.[0-9]+)(?:~(rc|beta)\.([0-9]+-[0-9]+))?') %} + {%- set parsed_version = datadog_install_settings.agent_version | regex_match('(([0-9]+)\.[0-9]+\.[0-9]+)(?:~(rc|beta)\.([0-9]+))?') %} {%- endif %} {# Determine defaults depending on specified version #} -{% if 'config_folder' not in datadog_settings %} - {%- if latest_agent_version or parsed_version[1] == '6' %} - {% do datadog_settings.update({'config_folder': '/etc/datadog-agent'}) %} - {%- else %} - {% do datadog_settings.update({'config_folder': '/etc/dd-agent'}) %} - {%- endif %} -{% endif %} +{%- if latest_agent_version or parsed_version[1] != '5' %} + {% do datadog_install_settings.update({'config_folder': '/etc/datadog-agent'}) %} +{%- else %} + {% do datadog_install_settings.update({'config_folder': '/etc/dd-agent'}) %} +{%- endif %} -{% if 'config_file' not in datadog_settings %} - {%- if latest_agent_version or parsed_version[1] == '6' %} - {% do datadog_settings.update({'config_file': 'datadog.yaml'}) %} - {%- else %} - {% do datadog_settings.update({'config_file': 'datadog.conf'}) %} - {%- endif %} -{% endif %} +{%- if latest_agent_version or parsed_version[1] != '5' %} + {% do datadog_install_settings.update({'config_file': 'datadog.yaml'}) %} +{%- else %} + {% do datadog_install_settings.update({'config_file': 'datadog.conf'}) %} +{%- endif %} -{% if 'checks_confd' not in datadog_settings %} - {%- if latest_agent_version or parsed_version[1] == '6' %} - {% do datadog_settings.update({'checks_confd': '/etc/datadog-agent/conf.d'}) %} +{%- if 'confd_path' in datadog_config %} + {% do datadog_install_settings.update({'confd_path': datadog_config.confd_path }) %} +{%- else %} + {%- if latest_agent_version or parsed_version[1] != '5' %} + {% do datadog_install_settings.update({'confd_path': '/etc/datadog-agent/conf.d'}) %} {%- else %} - {% do datadog_settings.update({'checks_confd': '/etc/dd-agent/conf.d'}) %} + {% do datadog_install_settings.update({'confd_path': '/etc/dd-agent/conf.d'}) %} {%- endif %} -{% endif %} +{%- endif %} diff --git a/datadog/service.sls b/datadog/service.sls index 66ac8f4..5a23b47 100644 --- a/datadog/service.sls +++ b/datadog/service.sls @@ -1,13 +1,13 @@ -{% from "datadog/map.jinja" import datadog_settings with context %} -{% set config_file_path = '%s/%s'|format(datadog_settings.config_folder, datadog_settings.config_file) -%} +{% from "datadog/map.jinja" import datadog_install_settings, datadog_checks with context %} +{% set config_file_path = '%s/%s'|format(datadog_install_settings.config_folder, datadog_install_settings.config_file) -%} datadog-agent-service: service.running: - - name: {{ datadog_settings.service_name }} + - name: datadog-agent - enable: True - watch: - - pkg: {{ datadog_settings.pkg_name }} + - pkg: datadog-agent - file: {{ config_file_path }} -{%- if datadog_settings.checks is defined %} - - file: {{ datadog_settings.checks_confd }}/* +{%- if datadog_checks | length %} + - file: {{ datadog_install_settings.confd_path }}/* {% endif %} diff --git a/datadog/uninstall.sls b/datadog/uninstall.sls index 0de4fc6..626e790 100644 --- a/datadog/uninstall.sls +++ b/datadog/uninstall.sls @@ -1,11 +1,11 @@ -{% from "datadog/map.jinja" import datadog_settings with context %} +{% from "datadog/map.jinja" import datadog_install_settings with context %} datadog-uninstall: service.dead: - - name: {{ datadog_settings.service_name }} + - name: datadog-agent - enable: False pkg.removed: - pkgs: - - {{ datadog_settings.pkg_name }} + - datadog-agent - require: - service: datadog-uninstall diff --git a/pillar.example b/pillar.example index 72f3fa9..538cfc2 100644 --- a/pillar.example +++ b/pillar.example @@ -1,16 +1,23 @@ datadog: - api_key: aaaaaaaabbbbbbbbccccccccdddddddd - site: datadoghq.com - python_version: 2 + config: + api_key: aaaaaaaabbbbbbbbccccccccdddddddd + site: datadoghq.com + python_version: 2 + checks: process: - init_config: - procfs_path: /proc - instances: - - name: ssh - search_string: ['sshd'] + config: + init_config: + procfs_path: /proc + instances: + - name: ssh + search_string: ['sshd'] tcp_check: - instances: - - host: 127.0.0.1 - name: sshd - port: 22 + config: + instances: + - host: 127.0.0.1 + name: sshd + port: 22 + + install_settings: + agent_version: latest diff --git a/test/pillar/datadog.sls b/test/pillar/datadog.sls index 74290f0..76d7782 100644 --- a/test/pillar/datadog.sls +++ b/test/pillar/datadog.sls @@ -1,9 +1,15 @@ datadog: - api_key: aaaaaaaabbbbbbbbccccccccdddddddd - site: datadoghq.com - python_version: 2 + config: + api_key: aaaaaaaabbbbbbbbccccccccdddddddd + site: datadoghq.com + python_version: 2 + checks: directory: - instances: - - directory: "/srv/pillar" - name: "pillars" + config: + instances: + - directory: "/srv/pillar" + name: "pillars" + + install_settings: + agent_version: latest diff --git a/test/start.sh b/test/start.sh index 552ebe2..893eb20 100644 --- a/test/start.sh +++ b/test/start.sh @@ -2,3 +2,7 @@ # Start masterless minion salt-call --local state.highstate -l debug + +echo "==== Done ====" + +sleep infinity