From 8714c2a39f71a8fee5d6f4bbea58f06cce6093b6 Mon Sep 17 00:00:00 2001 From: yogesh Date: Wed, 21 Jun 2017 13:50:47 -0500 Subject: [PATCH 01/30] adding option to install minimal collectd using ppa --- ansible.cfg | 9 +++++++++ defaults/main.yml | 5 +++-- inventory | 2 ++ tasks/install.ppa.yml | 7 ++++++- 4 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 ansible.cfg create mode 100644 inventory diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..f8b08af --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,9 @@ +[defaults] + +# disable SSH key host checking (no need to type 'yes' for every new VM) +host_key_checking = False + +hostfile = inventory +roles_path = ./roles:./roles_ext/ + +remote_user = vagrant \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index a26f9c9..b7e96bb 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -6,9 +6,10 @@ collectd_service: upstart collectd_service_autodetect: true # try to detect collectd_service # PPA options -collectd_use_ppa: no # Use the collectd PPA -collectd_use_ppa_latest: no # Don't fix package version to collectd_version +collectd_use_ppa: yes # Use the collectd PPA +collectd_use_ppa_latest: yes # Don't fix package version to collectd_version collectd_ppa_source: 'ppa:collectd/collectd-5.5' +collectd_use_ppa_minimal: yes # Source options collectd_prefix: /opt/collectd # The place where Collectd will be installed diff --git a/inventory b/inventory new file mode 100644 index 0000000..55f2674 --- /dev/null +++ b/inventory @@ -0,0 +1,2 @@ +[localhost] +localhost ansible_connection=local \ No newline at end of file diff --git a/tasks/install.ppa.yml b/tasks/install.ppa.yml index 0536f60..225ebbc 100644 --- a/tasks/install.ppa.yml +++ b/tasks/install.ppa.yml @@ -10,6 +10,11 @@ name=collectd={{ collectd_version }} when: not collectd_use_ppa_latest +- name: collectd-install | Install latest collectd minimal package + apt: name=collectd + install_recommends=no + when: collectd_use_ppa_latest and collectd_use_ppa_minimal + - name: collectd-install | Install latest collectd package apt: name=collectd - when: collectd_use_ppa_latest + when: collectd_use_ppa_latest and not collectd_use_ppa_minimal From 11e92cbfc23bf778785e9fd3a5466aef5d1454a3 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 12:32:05 -0500 Subject: [PATCH 02/30] adding amqp template --- defaults/main.yml | 10 ++++++++++ tasks/configure.yml | 4 ++++ templates/amqp.conf.j2 | 11 +++++++++++ 3 files changed, 25 insertions(+) create mode 100644 templates/amqp.conf.j2 diff --git a/defaults/main.yml b/defaults/main.yml index b7e96bb..fd3ea9c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -74,3 +74,13 @@ collectd_config_file: "{{ '/etc/collectd/collectd.conf' if collectd_use_ppa else collectd_config_dir: "{{ '/etc/collectd/collectd.conf.d' if collectd_use_ppa else collectd_plugins_prefix }}" collectd_plugins_dir: "{{ '/usr/lib/collectd' if collectd_use_ppa else collectd_prefix + '/lib/collectd' }}" collectd_sbin_path: "{{ '/usr/sbin' if collectd_use_ppa else collectd_prefix_sbin }}" + +collectd_write_amqp: yes +collectd_write_amqp_options: + - Host "indices-manager" + - Port "5672" + - User "indices" + - Password "manager" + - Exchange "collectd-exchange" + - RoutingKey "indices-perf-key" + \ No newline at end of file diff --git a/tasks/configure.yml b/tasks/configure.yml index 635037b..3b04021 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -10,3 +10,7 @@ - name: collectd-configure | Configure additional types template: src=types.db.j2 dest={{collectd_additional_types_db_path}} notify: collectd restart + +- name: collectd-configure | Configure Addiotnal Plugin AMQP + template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' + notify: collectd restart diff --git a/templates/amqp.conf.j2 b/templates/amqp.conf.j2 new file mode 100644 index 0000000..d1d2369 --- /dev/null +++ b/templates/amqp.conf.j2 @@ -0,0 +1,11 @@ + + + {% for name, value in collectd_write_amqp_options.items() %} + {{ name }} "{{ value }}" + {% endfor %} + VHost "/" + Persistent false + StoreRates true + Format "json" + + \ No newline at end of file From be1ae11b50aaf8c06af254e27e8ca6085cc215a1 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 12:57:35 -0500 Subject: [PATCH 03/30] adding amqp config --- defaults/main.yml | 14 +++++++------- tasks/configure.yml | 2 +- templates/amqp.conf.j2 | 2 +- templates/collectd.conf.j2 | 1 + 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index fd3ea9c..bb16938 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -56,7 +56,7 @@ collectd_write_graphite_options: # Setup write_graphite (https://collectd.org # Setup logs collectd_logpath: # If it is not empty, will be used logfile -collectd_loglevel: info +collectd_loglevel: notice collectd_logrotate: yes collectd_type: yes collectd_logrotate_options: @@ -77,10 +77,10 @@ collectd_sbin_path: "{{ '/usr/sbin' if collectd_use_ppa else collectd_prefix_sbi collectd_write_amqp: yes collectd_write_amqp_options: - - Host "indices-manager" - - Port "5672" - - User "indices" - - Password "manager" - - Exchange "collectd-exchange" - - RoutingKey "indices-perf-key" + Host: "indices-manager" + Port: "5672" + User: "indices" + Password: "manager" + Exchange: "collectd-exchange" + RoutingKey: "indices-perf-key" \ No newline at end of file diff --git a/tasks/configure.yml b/tasks/configure.yml index 3b04021..07fe214 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -12,5 +12,5 @@ notify: collectd restart - name: collectd-configure | Configure Addiotnal Plugin AMQP - template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' + template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf notify: collectd restart diff --git a/templates/amqp.conf.j2 b/templates/amqp.conf.j2 index d1d2369..39f748b 100644 --- a/templates/amqp.conf.j2 +++ b/templates/amqp.conf.j2 @@ -1,7 +1,7 @@ {% for name, value in collectd_write_amqp_options.items() %} - {{ name }} "{{ value }}" + {{ name }} "{{ value }}" {% endfor %} VHost "/" Persistent false diff --git a/templates/collectd.conf.j2 b/templates/collectd.conf.j2 index c6b3782..ad9a97a 100644 --- a/templates/collectd.conf.j2 +++ b/templates/collectd.conf.j2 @@ -17,6 +17,7 @@ LoadPlugin logfile LogLevel "{{collectd_loglevel}}" File "{{collectd_logpath}}" Timestamp true + PrintSeverity true {% else %} LoadPlugin syslog From 787c21f3b8ce0e2f680000c51a49e711718e2461 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 13:06:08 -0500 Subject: [PATCH 04/30] minor syntax --- templates/amqp.conf.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/amqp.conf.j2 b/templates/amqp.conf.j2 index 39f748b..d3a67af 100644 --- a/templates/amqp.conf.j2 +++ b/templates/amqp.conf.j2 @@ -3,9 +3,9 @@ {% for name, value in collectd_write_amqp_options.items() %} {{ name }} "{{ value }}" {% endfor %} - VHost "/" - Persistent false - StoreRates true - Format "json" + VHost "/" + Persistent false + StoreRates true + Format "json" \ No newline at end of file From 8ee93f0dea7d5f47d4db3e4943c0a96eeaea448c Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 13:47:19 -0500 Subject: [PATCH 05/30] eol correction --- templates/collectd.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/collectd.conf.j2 b/templates/collectd.conf.j2 index ad9a97a..197208e 100644 --- a/templates/collectd.conf.j2 +++ b/templates/collectd.conf.j2 @@ -65,4 +65,4 @@ LoadPlugin write_graphite Filter "*.conf" - + \ No newline at end of file From 9ace4afd2f45713f9978e3b426c1fafce1750c10 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 13:53:40 -0500 Subject: [PATCH 06/30] eol changes --- templates/amqp.conf.j2 | 2 +- templates/collectd.conf.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/amqp.conf.j2 b/templates/amqp.conf.j2 index d3a67af..16248b5 100644 --- a/templates/amqp.conf.j2 +++ b/templates/amqp.conf.j2 @@ -8,4 +8,4 @@ StoreRates true Format "json" - \ No newline at end of file + diff --git a/templates/collectd.conf.j2 b/templates/collectd.conf.j2 index 197208e..ad9a97a 100644 --- a/templates/collectd.conf.j2 +++ b/templates/collectd.conf.j2 @@ -65,4 +65,4 @@ LoadPlugin write_graphite Filter "*.conf" - \ No newline at end of file + From 14862ca58729e0b71a56f1780ce526feb740d83b Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 13:54:48 -0500 Subject: [PATCH 07/30] added validate --- tasks/configure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/configure.yml b/tasks/configure.yml index 07fe214..3b04021 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -12,5 +12,5 @@ notify: collectd restart - name: collectd-configure | Configure Addiotnal Plugin AMQP - template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf + template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart From 8cdcc71ef4285ee52deeaad7913d01fe5e13cb71 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 14:15:53 -0500 Subject: [PATCH 08/30] load plugin amqp --- templates/amqp.conf.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/amqp.conf.j2 b/templates/amqp.conf.j2 index 16248b5..0f95595 100644 --- a/templates/amqp.conf.j2 +++ b/templates/amqp.conf.j2 @@ -1,3 +1,4 @@ +LoadPlugin amqp {% for name, value in collectd_write_amqp_options.items() %} From 3954a23a77b9ec34c7ca369e2278cc826e57616d Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 3 Jul 2017 15:38:22 -0500 Subject: [PATCH 09/30] added cpu config --- defaults/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index bb16938..161692d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -36,6 +36,10 @@ collectd_default_plugins_options: interface: - Interface lo - IgnoreSelected true + cpu: + - ValuesPercentage true + - ReportByState false + - ReportByCpu false # Additional types # format: { name: ..., value: ... } From 5a7af3ca489b928074bf3c93eaa78d6d8d2b9bdb Mon Sep 17 00:00:00 2001 From: yogeshvu Date: Tue, 1 Aug 2017 21:46:55 -0500 Subject: [PATCH 10/30] added fqdn to true default --- defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 161692d..8a482e5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -22,7 +22,7 @@ collectd_plugins_prefix: "{{collectd_prefix_conf}}/conf.d" collectd_interval: 10 collectd_readthreads: 7 collectd_hostname: "{{ inventory_hostname }}" -collectd_fdqnlookup: false +collectd_fdqnlookup: true # Collectd plugins collectd_plugins: [] # Ex. [nginx, memcached] @@ -83,8 +83,8 @@ collectd_write_amqp: yes collectd_write_amqp_options: Host: "indices-manager" Port: "5672" - User: "indices" - Password: "manager" + User: "indices_user" + Password: "indices_manager" Exchange: "collectd-exchange" RoutingKey: "indices-perf-key" \ No newline at end of file From 8b09dc0df199a11305be2dcc31b930fd68e63cc3 Mon Sep 17 00:00:00 2001 From: yogesh Date: Sat, 12 Aug 2017 10:03:51 -0500 Subject: [PATCH 11/30] added the csv plugin --- defaults/main.yml | 5 +++++ templates/collectd.conf.j2 | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index 161692d..da226fe 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -71,6 +71,11 @@ collectd_logrotate_options: - rotate 7 - size 10M +# If the csv path is set the plugin will be enabled +#'/var/lib/collectd/csv' +collectd_csv_logdir: +collectd_csv_logdir_storerates: true + # Paths collectd_types_db_path: "{{ '/usr/share/collectd/types.db' if collectd_use_ppa else collectd_prefix + '/share/collectd/types.db' }}" collectd_additional_types_db_path: "{{ '/usr/share/collectd/additional-types.db' if collectd_use_ppa else collectd_prefix_type + '/types.db' }}" diff --git a/templates/collectd.conf.j2 b/templates/collectd.conf.j2 index ad9a97a..708bf6c 100644 --- a/templates/collectd.conf.j2 +++ b/templates/collectd.conf.j2 @@ -27,6 +27,18 @@ LoadPlugin syslog {% endif %} +# Logging CSV +#StoreRates true|false +#If set to true, convert counter values to rates. If set to false (the default) counter values are stored as is, i. e. as an increasing integer number. +{% if collectd_csv_logdir %} +LoadPlugin csv + + DataDir "{{collectd_csv_logdir}}" + StoreRates "{{collectd_csv_logdir_storerates|default('true')}}" + +{% endif %} + + # Default plugins {% for plugin in collectd_default_plugins %} LoadPlugin {{ plugin }} From 0a7295c9d742c72d21bfd0fa6b5ccb0dcf523e20 Mon Sep 17 00:00:00 2001 From: yogesh Date: Sat, 12 Aug 2017 11:09:37 -0500 Subject: [PATCH 12/30] update cpu settings --- defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index e75abcc..9e2a84c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -39,7 +39,9 @@ collectd_default_plugins_options: cpu: - ValuesPercentage true - ReportByState false - - ReportByCpu false + - ReportByCpu true + - ReportNumCpu true + # Additional types # format: { name: ..., value: ... } From ca3852d643f3cabec3bc1171b58d5287ea3316c2 Mon Sep 17 00:00:00 2001 From: yogesh Date: Sat, 12 Aug 2017 11:52:14 -0500 Subject: [PATCH 13/30] new collectd ci ppa --- README.md | 2 +- defaults/main.yml | 5 ++++- tasks/install.ppa.yml | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f02a7e0..2be3a49 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Ansible role which help you with: ```yaml collectd_enabled: yes # Enable the role -collectd_version: 5.5.2 # Set version +collectd_version: 5.7.2 # Set version # PPA options collectd_use_ppa: no # Use the collectd PPA diff --git a/defaults/main.yml b/defaults/main.yml index 9e2a84c..dc9c722 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,7 +1,7 @@ --- collectd_enabled: yes # Enable the role -collectd_version: 5.5.2 # Set version +collectd_version: 5.7.2 # Set version collectd_service: upstart collectd_service_autodetect: true # try to detect collectd_service @@ -11,6 +11,9 @@ collectd_use_ppa_latest: yes # Don't fix package version to collectd_ver collectd_ppa_source: 'ppa:collectd/collectd-5.5' collectd_use_ppa_minimal: yes +collectd_use_ppa_ci: yes +collectd_use_ppa_ci_codename: collectd-5.7 + # Source options collectd_prefix: /opt/collectd # The place where Collectd will be installed collectd_prefix_type: "{{collectd_prefix}}/etc" diff --git a/tasks/install.ppa.yml b/tasks/install.ppa.yml index 225ebbc..1d1e83c 100644 --- a/tasks/install.ppa.yml +++ b/tasks/install.ppa.yml @@ -4,6 +4,22 @@ apt_repository: > repo={{ collectd_ppa_source }} update_cache=yes + when: not collectd_use_ppa_ci + +- name: collectd-install | add signing key + apt_repository: + repo: deb http://pkg.ci.collectd.org/deb/ {{ansible_distribution_release}} {{collectd_use_ppa_ci_codename}} + state: present + filename: collectd-ci.list + update_cache: yes + when: collectd_use_ppa_ci + + +- name: collectd-install | add signing key + apt_key: + url: "http://pkg.ci.collectd.org/pubkey.asc" + state: present + when: collectd_use_ppa_ci - name: collectd-install | Install specific collectd package apt: > From b07d64ddfd71d6c77f274c301d3d58c562aecbd6 Mon Sep 17 00:00:00 2001 From: yogesh Date: Sat, 12 Aug 2017 11:57:59 -0500 Subject: [PATCH 14/30] keys added --- tasks/install.ppa.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tasks/install.ppa.yml b/tasks/install.ppa.yml index 1d1e83c..57b7212 100644 --- a/tasks/install.ppa.yml +++ b/tasks/install.ppa.yml @@ -5,21 +5,20 @@ repo={{ collectd_ppa_source }} update_cache=yes when: not collectd_use_ppa_ci + +- name: collectd-install | add signing key + apt_key: + url: "http://pkg.ci.collectd.org/pubkey.asc" + state: present + when: collectd_use_ppa_ci -- name: collectd-install | add signing key +- name: collectd-install | add ppa key apt_repository: repo: deb http://pkg.ci.collectd.org/deb/ {{ansible_distribution_release}} {{collectd_use_ppa_ci_codename}} state: present filename: collectd-ci.list update_cache: yes when: collectd_use_ppa_ci - - -- name: collectd-install | add signing key - apt_key: - url: "http://pkg.ci.collectd.org/pubkey.asc" - state: present - when: collectd_use_ppa_ci - name: collectd-install | Install specific collectd package apt: > From b482bd9a7cc8a68d155250ebe9ba8cfed990d129 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 14 Aug 2017 12:00:14 -0500 Subject: [PATCH 15/30] wip numa_cpu_aggregation --- defaults/main.yml | 12 ++++++++++ tasks/configure.yml | 4 ++++ templates/numa_cpu_aggreagtion.conf.j2 | 31 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 templates/numa_cpu_aggreagtion.conf.j2 diff --git a/defaults/main.yml b/defaults/main.yml index dc9c722..96f0e05 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -97,4 +97,16 @@ collectd_write_amqp_options: Password: "indices_manager" Exchange: "collectd-exchange" RoutingKey: "indices-perf-key" + + + +collectd_numa_cpu_aggr: no +collectd_numa_node_cpus: [ "0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38", + "1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39"] + +collectd_numa_cpu_aggr_options: + CalculateAverage: true + CalculateSum: false + + \ No newline at end of file diff --git a/tasks/configure.yml b/tasks/configure.yml index 3b04021..ddcd920 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -14,3 +14,7 @@ - name: collectd-configure | Configure Addiotnal Plugin AMQP template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart + +- name: collectd-configure | Configure Additional Plugin NUMA CPU Aggregation + template: src=numa_cpu_aggregation.conf.j2 dest={{collectd_config_dir}}/numa_cpu_aggregation.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' + notify: collectd restart \ No newline at end of file diff --git a/templates/numa_cpu_aggreagtion.conf.j2 b/templates/numa_cpu_aggreagtion.conf.j2 new file mode 100644 index 0000000..4e40e42 --- /dev/null +++ b/templates/numa_cpu_aggreagtion.conf.j2 @@ -0,0 +1,31 @@ +# Default plugins +#LoadPlugin cpu +# +# ValuesPercentage true +# ReportByState false +# ReportByCpu true +# ReportNumCpu true +# + +{% if collectd_numa_cpu_aggr %} +LoadPlugin aggregation + + {% for node_num in collectd_numa_node_cpus.items() %} +  +    Plugin "cpu" +    Type "percent" +    #PluginInstance "/[0,1]$/" + PluginInstance "/[{{node_num}}]$/" +    GroupBy "Host" +    GroupBy "TypeInstance" +    #SetPlugin "ycpu-one" name of the plugin for which the data will be published +    SetPlugin "numa_group_{{loop.index}}" +   # SetPluginInstance "%{aggregation}" +    #CalculateSum true + {% for name, value in collectd_numa_cpu_aggr_options.items() %} + {{ name }} "{{ value }}" + {% endfor %} +    #CalculateAverage true +  + {% endfor %} +{% endif %} From 4d541a14428064f05e7d5e80d0d94d8eb5119dba Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 14 Aug 2017 12:10:46 -0500 Subject: [PATCH 16/30] renemed conf file --- ...{numa_cpu_aggreagtion.conf.j2 => numa_cpu_aggregation.conf.j2} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename templates/{numa_cpu_aggreagtion.conf.j2 => numa_cpu_aggregation.conf.j2} (100%) diff --git a/templates/numa_cpu_aggreagtion.conf.j2 b/templates/numa_cpu_aggregation.conf.j2 similarity index 100% rename from templates/numa_cpu_aggreagtion.conf.j2 rename to templates/numa_cpu_aggregation.conf.j2 From fdfbd5b2c96ca59f8bc7555a51e710253a266931 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 14 Aug 2017 13:46:11 -0500 Subject: [PATCH 17/30] added more updates to aggregation --- defaults/main.yml | 2 +- templates/numa_cpu_aggregation.conf.j2 | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 96f0e05..5264fed 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -100,7 +100,7 @@ collectd_write_amqp_options: -collectd_numa_cpu_aggr: no +collectd_numa_cpu_aggr: yes collectd_numa_node_cpus: [ "0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38", "1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39"] diff --git a/templates/numa_cpu_aggregation.conf.j2 b/templates/numa_cpu_aggregation.conf.j2 index 4e40e42..f05212e 100644 --- a/templates/numa_cpu_aggregation.conf.j2 +++ b/templates/numa_cpu_aggregation.conf.j2 @@ -10,22 +10,16 @@ {% if collectd_numa_cpu_aggr %} LoadPlugin aggregation - {% for node_num in collectd_numa_node_cpus.items() %} -  +{% for node_num in collectd_numa_node_cpus %} +     Plugin "cpu"     Type "percent" -    #PluginInstance "/[0,1]$/" - PluginInstance "/[{{node_num}}]$/" +   PluginInstance "/[{{ node_num }}]$/"     GroupBy "Host"     GroupBy "TypeInstance" -    #SetPlugin "ycpu-one" name of the plugin for which the data will be published     SetPlugin "numa_group_{{loop.index}}" -   # SetPluginInstance "%{aggregation}" -    #CalculateSum true {% for name, value in collectd_numa_cpu_aggr_options.items() %} - {{ name }} "{{ value }}" + {{ name }} {{ value }} {% endfor %} -    #CalculateAverage true -  - {% endfor %} -{% endif %} +{% endfor %} +{% endif %} \ No newline at end of file From 1483875c7bfe500d16066f0ca9ff681f84ba5fdd Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 14 Aug 2017 14:59:18 -0500 Subject: [PATCH 18/30] added updates to the aggregation plugin --- ansible.cfg | 3 ++- defaults/main.yml | 4 ++-- templates/numa_cpu_aggregation.conf.j2 | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ansible.cfg b/ansible.cfg index f8b08af..c38090e 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -6,4 +6,5 @@ host_key_checking = False hostfile = inventory roles_path = ./roles:./roles_ext/ -remote_user = vagrant \ No newline at end of file +remote_user = vagrant +log_path = $HOME/ansible.log \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index 5264fed..ad3fc10 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -105,8 +105,8 @@ collectd_numa_node_cpus: [ "0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 "1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39"] collectd_numa_cpu_aggr_options: - CalculateAverage: true - CalculateSum: false + CalculateAverage: True + CalculateSum: False \ No newline at end of file diff --git a/templates/numa_cpu_aggregation.conf.j2 b/templates/numa_cpu_aggregation.conf.j2 index f05212e..faa5734 100644 --- a/templates/numa_cpu_aggregation.conf.j2 +++ b/templates/numa_cpu_aggregation.conf.j2 @@ -17,9 +17,11 @@ LoadPlugin aggregation    PluginInstance "/[{{ node_num }}]$/"     GroupBy "Host"     GroupBy "TypeInstance" -    SetPlugin "numa_group_{{loop.index}}" +    SetPlugin "numa-group-{{loop.index}}" {% for name, value in collectd_numa_cpu_aggr_options.items() %} {{ name }} {{ value }} {% endfor %} -{% endfor %} -{% endif %} \ No newline at end of file + +{% endfor %} + +{% endif %} From cf6e35f695ac7a8a68db6f7e8c272d4a6a61aa74 Mon Sep 17 00:00:00 2001 From: yogesh Date: Mon, 14 Aug 2017 16:08:54 -0500 Subject: [PATCH 19/30] updates to true/false --- defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index ad3fc10..5264fed 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -105,8 +105,8 @@ collectd_numa_node_cpus: [ "0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 "1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39"] collectd_numa_cpu_aggr_options: - CalculateAverage: True - CalculateSum: False + CalculateAverage: true + CalculateSum: false \ No newline at end of file From 2143870f75f4ea80d15e2c0876b7dcb4d6f31adf Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Tue, 10 Oct 2017 17:23:24 +0000 Subject: [PATCH 20/30] added support for docker monitoring --- defaults/main.yml | 2 ++ tasks/configure.yml | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/defaults/main.yml b/defaults/main.yml index 5264fed..a3f2b6f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -109,4 +109,6 @@ collectd_numa_cpu_aggr_options: CalculateSum: false +# Enable docker monitoring by default +docker_stats: yes \ No newline at end of file diff --git a/tasks/configure.yml b/tasks/configure.yml index ddcd920..a59f586 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -17,4 +17,22 @@ - name: collectd-configure | Configure Additional Plugin NUMA CPU Aggregation template: src=numa_cpu_aggregation.conf.j2 dest={{collectd_config_dir}}/numa_cpu_aggregation.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' - notify: collectd restart \ No newline at end of file + notify: collectd restart + +- name: collectd-configure | Configure Additional Plugin Docker Stats 1 of 3 + git: + dest: /usr/share/collectd/docker-collectd-plugin + repo: "https://github.com/lebauce/docker-collectd-plugin.git" + when: docker_stats + + +- name: collectd-configure | Configure Additional Plugin Docker Stats 2 of 3 + shell: pip install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt + notify: collectd restart + when: docker_stats + + +- name: collectd-configure | Configure Additional Plugin Docker Stats 3 of 3 + template: src=docker.conf.j2 dest={{collectd_config_dir}}/docker.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' + notify: collectd restart + when: docker_stats \ No newline at end of file From 3bb3885c121222da87d36e327291fcb8579969f0 Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Tue, 10 Oct 2017 17:27:47 +0000 Subject: [PATCH 21/30] adding template docker config file --- templates/docker.conf.j2 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 templates/docker.conf.j2 diff --git a/templates/docker.conf.j2 b/templates/docker.conf.j2 new file mode 100644 index 0000000..1ef184e --- /dev/null +++ b/templates/docker.conf.j2 @@ -0,0 +1,12 @@ +{% if docker_stats %} +TypesDB "/usr/share/collectd/docker-collectd-plugin/dockerplugin.db" +LoadPlugin python + + ModulePath "/usr/share/collectd/docker-collectd-plugin" + Import "dockerplugin" + + BaseURL "unix://var/run/docker.sock" + Timeout 3 + + +{% endif %} \ No newline at end of file From a6fc37e6598fa6c27c6fee68be95cdb73564e91a Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Wed, 11 Oct 2017 14:44:20 +0000 Subject: [PATCH 22/30] added flag to check if the plugins are enable/disabl before populating the config file --- tasks/configure.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/configure.yml b/tasks/configure.yml index a59f586..4763996 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -14,10 +14,12 @@ - name: collectd-configure | Configure Addiotnal Plugin AMQP template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart + when: collectd_write_amqp - name: collectd-configure | Configure Additional Plugin NUMA CPU Aggregation template: src=numa_cpu_aggregation.conf.j2 dest={{collectd_config_dir}}/numa_cpu_aggregation.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart + when: collectd_numa_cpu_aggr - name: collectd-configure | Configure Additional Plugin Docker Stats 1 of 3 git: @@ -25,7 +27,6 @@ repo: "https://github.com/lebauce/docker-collectd-plugin.git" when: docker_stats - - name: collectd-configure | Configure Additional Plugin Docker Stats 2 of 3 shell: pip install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt notify: collectd restart From 58b25ab1a1b2fb05cc1f463b2feb93c9c5fcf91d Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Wed, 11 Oct 2017 14:55:40 +0000 Subject: [PATCH 23/30] fixing the boolean error in collectd --- templates/numa_cpu_aggregation.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/numa_cpu_aggregation.conf.j2 b/templates/numa_cpu_aggregation.conf.j2 index faa5734..a1a2bf2 100644 --- a/templates/numa_cpu_aggregation.conf.j2 +++ b/templates/numa_cpu_aggregation.conf.j2 @@ -19,7 +19,7 @@ LoadPlugin aggregation     GroupBy "TypeInstance"     SetPlugin "numa-group-{{loop.index}}" {% for name, value in collectd_numa_cpu_aggr_options.items() %} - {{ name }} {{ value }} + {{ name }} {{ value| lower }} {% endfor %} {% endfor %} From a296ee4095025c39eb371617b623e966dd0ed1cc Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Thu, 12 Oct 2017 19:29:57 +0000 Subject: [PATCH 24/30] added support for the intel_rdt intel_pmutools collectd compile from source --- defaults/main.yml | 22 +++- tasks/collectd.yml | 8 +- tasks/configure.yml | 14 ++- tasks/install.git.yml | 32 ++++++ templates/initd.collectd.j2 | 200 +++++++++++++++++++++++++++++++++ templates/intelpmutool.conf.j2 | 8 ++ templates/intelrdt.conf.j2 | 9 ++ 7 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 tasks/install.git.yml create mode 100755 templates/initd.collectd.j2 create mode 100644 templates/intelpmutool.conf.j2 create mode 100644 templates/intelrdt.conf.j2 diff --git a/defaults/main.yml b/defaults/main.yml index a3f2b6f..b9e79ff 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -6,7 +6,7 @@ collectd_service: upstart collectd_service_autodetect: true # try to detect collectd_service # PPA options -collectd_use_ppa: yes # Use the collectd PPA +collectd_use_ppa: no # Use the collectd PPA collectd_use_ppa_latest: yes # Don't fix package version to collectd_version collectd_ppa_source: 'ppa:collectd/collectd-5.5' collectd_use_ppa_minimal: yes @@ -15,6 +15,8 @@ collectd_use_ppa_ci: yes collectd_use_ppa_ci_codename: collectd-5.7 # Source options +collectd_use_git: yes +collectd_prefix_git_src: /home/ubuntu/collectd collectd_prefix: /opt/collectd # The place where Collectd will be installed collectd_prefix_type: "{{collectd_prefix}}/etc" collectd_prefix_conf: "{{collectd_prefix}}/etc" @@ -111,4 +113,20 @@ collectd_numa_cpu_aggr_options: # Enable docker monitoring by default docker_stats: yes - \ No newline at end of file + + + +# Enable docker monitoring by default +collectd_intel_pmutools: yes +pmutools_event_dir: "" +collectd_intel_pmutools_options: + ReportHardwareCacheEvents: true + ReportKernelPMUEvents: true + ReportSoftwareEvents: true + HardwareEvents: "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_$" + +collectd_intel_rdt: no +collectd_intel_rdt_interval: 1 +collectd_intel_rdt_options: + Cores: "" + diff --git a/tasks/collectd.yml b/tasks/collectd.yml index 6eb7f8b..7496390 100644 --- a/tasks/collectd.yml +++ b/tasks/collectd.yml @@ -14,9 +14,15 @@ tags: [collectd, collectd-install] - include: install.deb.yml - when: not collectd_use_ppa and ansible_os_family == 'Debian' + when: not collectd_use_ppa and not collectd_use_git and ansible_os_family == 'Debian' tags: [collectd, collectd-install] + +- include: install.git.yml + when: not collectd_use_ppa and collectd_use_git and ansible_os_family == 'Debian' + tags: [collectd, collectd-install] + + - include: configure.yml tags: [collectd, collectd-configure] diff --git a/tasks/configure.yml b/tasks/configure.yml index 4763996..d66ca43 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -32,8 +32,18 @@ notify: collectd restart when: docker_stats - - name: collectd-configure | Configure Additional Plugin Docker Stats 3 of 3 template: src=docker.conf.j2 dest={{collectd_config_dir}}/docker.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart - when: docker_stats \ No newline at end of file + when: docker_stats + +- name: collectd-configure | Configure Additional Plugin intel_pmutools + template: src=intelpmutool.conf.j2 dest={{collectd_config_dir}}/intelpmutool.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' + notify: collectd restart + when: collectd_intel_pmutools + +- name: collectd-configure | Configure Additional Plugin intel_rdt + template: src=intelrdt.conf.j2 dest={{collectd_config_dir}}/intelrdt.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' + notify: collectd restart + when: collectd_intel_rdt + \ No newline at end of file diff --git a/tasks/install.git.yml b/tasks/install.git.yml new file mode 100644 index 0000000..ec8f820 --- /dev/null +++ b/tasks/install.git.yml @@ -0,0 +1,32 @@ +--- + +- name: collectd-install | Install dependencies + apt: name={{item}} force=yes + with_items: [build-essential, libcurl4-openssl-dev, libpcap-dev, libyajl-dev] + +- name: collectd-install | Download Collectd + git: + dest: "{{ collectd_prefix_git_src }}" + repo: "https://github.com/collectd/collectd.git" + become: yes + register: collectd_source + +- name: collectd-install | Install Collectd + shell: ./configure --prefix={{collectd_prefix}} && make && make all install + args: + chdir: "{{ collectd_prefix_git_src }}" + when: collectd_source.changed + become: yes + +- name: collectd-install | Create /etc/init.d/colelctd + template: src=init.collectd.j2 dest=/etc/init.d/collectd + become: yes + + +- name: collectd-install | Enable collectd.service + shell: systemctl enable collectd.service + become: yes + +- name: collectd-install | start the collectd.service + service: name=collectd state=started enabled=yes + become: yes diff --git a/templates/initd.collectd.j2 b/templates/initd.collectd.j2 new file mode 100755 index 0000000..74f8732 --- /dev/null +++ b/templates/initd.collectd.j2 @@ -0,0 +1,200 @@ +#! /bin/bash +# +# collectd - start and stop the statistics collection daemon +# http://collectd.org/ +# +# Copyright (C) 2005-2006 Florian Forster +# Copyright (C) 2006-2009 Sebastian Harl +# + +### BEGIN INIT INFO +# Provides: collectd +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# Should-Start: $network $named $syslog $time cpufrequtils +# Should-Stop: $network $named $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: manage the statistics collection daemon +# Description: collectd is the statistics collection daemon. +# It is a small daemon which collects system information +# periodically and provides mechanisms to monitor and store +# the values in a variety of ways. +### END INIT INFO + +. /lib/lsb/init-functions + +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/opt/collectd/sbin:/opt/collectd:"{{collectd_prefix}}":"{{collectd_prefix}}"/sbin + +DISABLE=0 + +DESC="statistics collection and monitoring daemon" +NAME=collectd +DAEMON="{{collectd_prefix_sbin}}"/collectd + +CONFIGFILE="{{collectd_prefix_conf}}"/collectd.conf +PIDFILE="{{collectd_prefix}}"/run/collectd.pid + +USE_COLLECTDMON=1 +COLLECTDMON_DAEMON="{{collectd_prefix_sbin}}"/collectdmon + +MAXWAIT=30 + +# Gracefully exit if the package has been removed. +test -x $DAEMON || exit 0 + +if [ -r /etc/default/$NAME ]; then + . /etc/default/$NAME +fi + +if test "$ENABLE_COREFILES" == 1; then + ulimit -c unlimited +fi + +# return: +# 0 if config is fine +# 1 if there is a syntax error +# 2 if there is no configuration +check_config() { + if test ! -e "$CONFIGFILE"; then + return 2 + fi + if ! $DAEMON -t -C "$CONFIGFILE"; then + return 1 + fi + return 0 +} + +# return: +# 0 if the daemon has been started +# 1 if the daemon was already running +# 2 if the daemon could not be started +# 3 if the daemon was not supposed to be started +d_start() { + if test "$DISABLE" != 0; then + # we get here during restart + log_progress_msg "disabled by /etc/default/$NAME" + return 3 + fi + + if test ! -e "$CONFIGFILE"; then + # we get here during restart + log_progress_msg "disabled, no configuration ($CONFIGFILE) found" + return 3 + fi + + check_config + rc="$?" + if test "$rc" -ne 0; then + log_progress_msg "not starting, configuration error" + return 2 + fi + + if test "$USE_COLLECTDMON" == 1; then + start-stop-daemon --start --quiet --oknodo --pidfile "$PIDFILE" \ + --exec $COLLECTDMON_DAEMON -- -P "$PIDFILE" -- -C "$CONFIGFILE" \ + || return 2 + else + start-stop-daemon --start --quiet --oknodo --pidfile "$PIDFILE" \ + --exec $DAEMON -- -C "$CONFIGFILE" -P "$PIDFILE" \ + || return 2 + fi + return 0 +} + +still_running_warning=" +WARNING: $NAME might still be running. +In large setups it might take some time to write all pending data to +the disk. You can adjust the waiting time in /etc/default/collectd." + +# return: +# 0 if the daemon has been stopped +# 1 if the daemon was already stopped +# 2 if daemon could not be stopped +d_stop() { + PID=$( cat "$PIDFILE" 2> /dev/null ) || true + + start-stop-daemon --stop --quiet --oknodo --pidfile "$PIDFILE" + rc="$?" + + if test "$rc" -eq 2; then + return 2 + fi + + sleep 1 + if test -n "$PID" && kill -0 $PID 2> /dev/null; then + i=0 + while kill -0 $PID 2> /dev/null; do + i=$(( $i + 2 )) + echo -n " ." + + if test $i -gt $MAXWAIT; then + log_progress_msg "$still_running_warning" + return 2 + fi + + sleep 2 + done + return "$rc" + fi + return "$rc" +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + d_start + case "$?" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + 3) log_end_msg 255; true ;; + *) log_end_msg 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + d_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + esac + ;; + status) + status_of_proc -p "$PIDFILE" "$DAEMON" "$NAME" && exit 0 || exit $? + ;; + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + check_config + rc="$?" + if test "$rc" -eq 1; then + log_progress_msg "not restarting, configuration error" + log_end_msg 1 + exit 1 + fi + d_stop + rc="$?" + case "$rc" in + 0|1) + sleep 1 + d_start + rc2="$?" + case "$rc2" in + 0|1) log_end_msg 0 ;; + 2) log_end_msg 1 ;; + 3) log_end_msg 255; true ;; + *) log_end_msg 1 ;; + esac + ;; + *) + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $0 {start|stop|restart|force-reload|status}" >&2 + exit 3 + ;; +esac + +# vim: syntax=sh noexpandtab sw=4 ts=4 : + diff --git a/templates/intelpmutool.conf.j2 b/templates/intelpmutool.conf.j2 new file mode 100644 index 0000000..e575d4f --- /dev/null +++ b/templates/intelpmutool.conf.j2 @@ -0,0 +1,8 @@ +LoadPlugin intel_pmu + + + {% for name, value in collectd_intel_pmutools_options.items() %} + {{ name }} "{{ value|lower }}" + {% endfor %} + EventList "{{ pmutools_event_dir }}" + diff --git a/templates/intelrdt.conf.j2 b/templates/intelrdt.conf.j2 new file mode 100644 index 0000000..3f1555e --- /dev/null +++ b/templates/intelrdt.conf.j2 @@ -0,0 +1,9 @@ + + Interval "{{collectd_intel_rdt_interval }}"" + + + + {% for name, value in collectd_intel_rdt_options.items() %} + {{ name }} "{{ value|lower }}" + {% endfor %} + Date: Sun, 15 Oct 2017 15:00:48 +0000 Subject: [PATCH 25/30] adding support for intel pmu-tools and intel cmt-rdt plugin --- defaults/main.yml | 2 +- tasks/configure.yml | 5 +++++ tasks/install.git.yml | 2 +- tasks/install.ppa.yml | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index b9e79ff..f2176b0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -117,7 +117,7 @@ docker_stats: yes # Enable docker monitoring by default -collectd_intel_pmutools: yes +collectd_intel_pmutools: no pmutools_event_dir: "" collectd_intel_pmutools_options: ReportHardwareCacheEvents: true diff --git a/tasks/configure.yml b/tasks/configure.yml index d66ca43..c3cc11e 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -37,6 +37,11 @@ notify: collectd restart when: docker_stats +- debug: + var: pmutools_event_dir + verbosity: 2 + when: collectd_intel_pmutools + - name: collectd-configure | Configure Additional Plugin intel_pmutools template: src=intelpmutool.conf.j2 dest={{collectd_config_dir}}/intelpmutool.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart diff --git a/tasks/install.git.yml b/tasks/install.git.yml index ec8f820..a9c04ad 100644 --- a/tasks/install.git.yml +++ b/tasks/install.git.yml @@ -18,7 +18,7 @@ when: collectd_source.changed become: yes -- name: collectd-install | Create /etc/init.d/colelctd +- name: collectd-install | Create /etc/init.d/collectd template: src=init.collectd.j2 dest=/etc/init.d/collectd become: yes diff --git a/tasks/install.ppa.yml b/tasks/install.ppa.yml index 57b7212..274bb09 100644 --- a/tasks/install.ppa.yml +++ b/tasks/install.ppa.yml @@ -20,6 +20,7 @@ update_cache: yes when: collectd_use_ppa_ci +# TODO: refine this to decide only based on recommended packages or not - name: collectd-install | Install specific collectd package apt: > name=collectd={{ collectd_version }} From 82e089089696a34e072fcdf6156fee30dae80729 Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Sun, 15 Oct 2017 17:02:52 +0000 Subject: [PATCH 26/30] adding intelrdt support --- defaults/main.yml | 3 ++- tasks/configure.yml | 22 +++++++++++++++++++--- tasks/install.git.yml | 16 ++++++++++++---- templates/intelrdt.conf.j2 | 4 ++-- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index f2176b0..94b9cf0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -119,11 +119,12 @@ docker_stats: yes # Enable docker monitoring by default collectd_intel_pmutools: no pmutools_event_dir: "" +pmu_tools_dir: "/opt/pmu-tools" collectd_intel_pmutools_options: ReportHardwareCacheEvents: true ReportKernelPMUEvents: true ReportSoftwareEvents: true - HardwareEvents: "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_$" + HardwareEvents: "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS,L2_RQSTS.ALL_$" collectd_intel_rdt: no collectd_intel_rdt_interval: 1 diff --git a/tasks/configure.yml b/tasks/configure.yml index c3cc11e..a3820f3 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -37,12 +37,28 @@ notify: collectd restart when: docker_stats -- debug: +- name: collectd-configure | Configure Additional Plugin intel_pmutools 1 of 4 + shell: cat "{{ pmu_tools_dir }}"/eventlist_info.txt + become: yes + register: event_dir + when: collectd_intel_pmutools + +- name: collectd-configure | Configure Additional Plugin intel_pmutools 2 of 4 + set_fact: + pmutools_event_dir: "{{ event_dir.stdout|default([]) }}" + when: collectd_intel_pmutools + +- name: collectd-configure | Configure Additional Plugin intel_pmutools 3 of 4 + debug: var: pmutools_event_dir - verbosity: 2 + verbosity: 4 when: collectd_intel_pmutools -- name: collectd-configure | Configure Additional Plugin intel_pmutools +# - name: collectd-configure | Configure Additional Plugin intel_pmutools 4 of 5 +# var: pmutools_event_dir +# when: collectd_intel_pmutools + +- name: collectd-configure | Configure Additional Plugin intel_pmutools 4 of 4 template: src=intelpmutool.conf.j2 dest={{collectd_config_dir}}/intelpmutool.conf validate='{{collectd_sbin_path}}/collectd -t -C %s' notify: collectd restart when: collectd_intel_pmutools diff --git a/tasks/install.git.yml b/tasks/install.git.yml index a9c04ad..ef669ad 100644 --- a/tasks/install.git.yml +++ b/tasks/install.git.yml @@ -2,7 +2,7 @@ - name: collectd-install | Install dependencies apt: name={{item}} force=yes - with_items: [build-essential, libcurl4-openssl-dev, libpcap-dev, libyajl-dev] + with_items: [build-essential, libcurl4-openssl-dev, libpcap-dev, libyajl-dev, flex, bison, pkg-config, automake] - name: collectd-install | Download Collectd git: @@ -12,16 +12,24 @@ register: collectd_source - name: collectd-install | Install Collectd - shell: ./configure --prefix={{collectd_prefix}} && make && make all install + shell: ./build.sh && ./configure --prefix={{collectd_prefix}} && make && make all install args: chdir: "{{ collectd_prefix_git_src }}" - when: collectd_source.changed + # when: collectd_source.changed become: yes - name: collectd-install | Create /etc/init.d/collectd - template: src=init.collectd.j2 dest=/etc/init.d/collectd + template: src=initd.collectd.j2 dest=/etc/init.d/collectd become: yes +- name: collectd-install | create run/collectd.pid file + shell: mkdir -p "{{collectd_prefix}}/run" + becom: yes + + +- name: collectd-install | make it executable /etc/init.d/collectd + shell: chmod +x /etc/init.d/collectd + become: yes - name: collectd-install | Enable collectd.service shell: systemctl enable collectd.service diff --git a/templates/intelrdt.conf.j2 b/templates/intelrdt.conf.j2 index 3f1555e..2e0435a 100644 --- a/templates/intelrdt.conf.j2 +++ b/templates/intelrdt.conf.j2 @@ -1,9 +1,9 @@ - Interval "{{collectd_intel_rdt_interval }}"" + Interval "{{collectd_intel_rdt_interval }}" {% for name, value in collectd_intel_rdt_options.items() %} {{ name }} "{{ value|lower }}" {% endfor %} - From cfae2377214ea78ac20153cd88d064c18ab767b0 Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Fri, 20 Oct 2017 14:09:13 +0000 Subject: [PATCH 27/30] updated dockerplugin directory --- defaults/main.yml | 1 + tasks/collectd.yml | 6 +++--- tasks/configure.yml | 8 +++++--- tasks/install.git.yml | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 94b9cf0..7bbf690 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,6 @@ --- +install_collectd: yes collectd_enabled: yes # Enable the role collectd_version: 5.7.2 # Set version collectd_service: upstart diff --git a/tasks/collectd.yml b/tasks/collectd.yml index 7496390..cd19105 100644 --- a/tasks/collectd.yml +++ b/tasks/collectd.yml @@ -10,16 +10,16 @@ when: ansible_os_family == 'Debian' and collectd_service_autodetect - include: install.ppa.yml - when: collectd_use_ppa + when: install_collectd and collectd_use_ppa tags: [collectd, collectd-install] - include: install.deb.yml - when: not collectd_use_ppa and not collectd_use_git and ansible_os_family == 'Debian' + when: install_collectd and not collectd_use_ppa and not collectd_use_git and ansible_os_family == 'Debian' tags: [collectd, collectd-install] - include: install.git.yml - when: not collectd_use_ppa and collectd_use_git and ansible_os_family == 'Debian' + when: install_collectd and not collectd_use_ppa and collectd_use_git and ansible_os_family == 'Debian' tags: [collectd, collectd-install] diff --git a/tasks/configure.yml b/tasks/configure.yml index a3820f3..436f102 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -24,12 +24,14 @@ - name: collectd-configure | Configure Additional Plugin Docker Stats 1 of 3 git: dest: /usr/share/collectd/docker-collectd-plugin - repo: "https://github.com/lebauce/docker-collectd-plugin.git" + #repo: "https://github.com/lebauce/docker-collectd-plugin.git" + #repo: "https://github.com/signalfx/docker-collectd-plugin.git" + repo: "https://github.com/doc-vu/docker-collectd-plugin.git" when: docker_stats - name: collectd-configure | Configure Additional Plugin Docker Stats 2 of 3 - shell: pip install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt - notify: collectd restart + shell: pip3 install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt && pip install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt + # notify: collectd restart when: docker_stats - name: collectd-configure | Configure Additional Plugin Docker Stats 3 of 3 diff --git a/tasks/install.git.yml b/tasks/install.git.yml index ef669ad..56675e1 100644 --- a/tasks/install.git.yml +++ b/tasks/install.git.yml @@ -12,7 +12,7 @@ register: collectd_source - name: collectd-install | Install Collectd - shell: ./build.sh && ./configure --prefix={{collectd_prefix}} && make && make all install + shell: ./build.sh && ./configure --prefix={{collectd_prefix}} --with-python LIBPYTHON_CPPFLAGS=/usr/include/python2.7 LIBPYTHON_LDFLAGS=/usr/local/lib/python2.7 PYTHON_CONFIG=/usr/bin/python2.7-config && make -j16 && make all install -j16 args: chdir: "{{ collectd_prefix_git_src }}" # when: collectd_source.changed From b10cd3ab70fd20e15838bf9642e5825d22412530 Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Fri, 20 Oct 2017 17:36:02 +0000 Subject: [PATCH 28/30] update --- tasks/install.git.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/install.git.yml b/tasks/install.git.yml index 56675e1..99fb415 100644 --- a/tasks/install.git.yml +++ b/tasks/install.git.yml @@ -17,6 +17,7 @@ chdir: "{{ collectd_prefix_git_src }}" # when: collectd_source.changed become: yes + ignore_error: yes - name: collectd-install | Create /etc/init.d/collectd template: src=initd.collectd.j2 dest=/etc/init.d/collectd From 8eae266e498a401f6c466215f3f8f65bfb0e06ff Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Fri, 20 Oct 2017 19:11:18 +0000 Subject: [PATCH 29/30] added force=yes git clone --- tasks/configure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tasks/configure.yml b/tasks/configure.yml index 436f102..f97d8d2 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -27,6 +27,7 @@ #repo: "https://github.com/lebauce/docker-collectd-plugin.git" #repo: "https://github.com/signalfx/docker-collectd-plugin.git" repo: "https://github.com/doc-vu/docker-collectd-plugin.git" + force: yes when: docker_stats - name: collectd-configure | Configure Additional Plugin Docker Stats 2 of 3 From 47be8eadc48c32d449d1a2d133a72f2567714164 Mon Sep 17 00:00:00 2001 From: Yogesh Barve Date: Tue, 24 Oct 2017 21:14:13 +0000 Subject: [PATCH 30/30] added the plugininstance to the numa-group --- templates/numa_cpu_aggregation.conf.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/numa_cpu_aggregation.conf.j2 b/templates/numa_cpu_aggregation.conf.j2 index a1a2bf2..1b67891 100644 --- a/templates/numa_cpu_aggregation.conf.j2 +++ b/templates/numa_cpu_aggregation.conf.j2 @@ -17,7 +17,8 @@ LoadPlugin aggregation    PluginInstance "/[{{ node_num }}]$/"     GroupBy "Host"     GroupBy "TypeInstance" -    SetPlugin "numa-group-{{loop.index}}" +    SetPlugin "numa-group" + SetPluginInstance "{{loop.index}}" {% for name, value in collectd_numa_cpu_aggr_options.items() %} {{ name }} {{ value| lower }} {% endfor %}