adding option to install minimal collectd using ppa

README.md

@@ -12,7 +12,7 @@ Ansible role which help you with:
 
 ```yaml
 collectd_enabled: yes               # Enable the role
-collectd_version: 5.5.2             # Set version
+collectd_version: 5.7.2             # Set version
 
 # PPA options
 collectd_use_ppa: no                # Use the collectd PPA

ansible.cfg

@@ -0,0 +1,10 @@
+[defaults]
+
+# disable SSH key host checking (no need to type 'yes' for every new VM)
+host_key_checking = False
+
+hostfile = inventory
+roles_path = ./roles:./roles_ext/
+
+remote_user = vagrant
+log_path = $HOME/ansible.log

defaults/main.yml

@@ -1,16 +1,23 @@
 ---
 
+install_collectd: yes
 collectd_enabled: yes               # Enable the role
-collectd_version: 5.5.2             # Set version
+collectd_version: 5.7.2             # Set version
 collectd_service: upstart
 collectd_service_autodetect: true   # try to detect collectd_service
 
 # PPA options
 collectd_use_ppa: no                # Use the collectd PPA
-collectd_use_ppa_latest: no         # Don't fix package version to collectd_version
+collectd_use_ppa_latest: yes         # Don't fix package version to collectd_version
 collectd_ppa_source: 'ppa:collectd/collectd-5.5'
+collectd_use_ppa_minimal: yes
+
+collectd_use_ppa_ci: yes
+collectd_use_ppa_ci_codename: collectd-5.7
 
 # Source options
+collectd_use_git: yes
+collectd_prefix_git_src: /home/ubuntu/collectd
 collectd_prefix: /opt/collectd      # The place where Collectd will be installed
 collectd_prefix_type: "{{collectd_prefix}}/etc"
 collectd_prefix_conf: "{{collectd_prefix}}/etc"
@@ -21,7 +28,7 @@ collectd_plugins_prefix: "{{collectd_prefix_conf}}/conf.d"
 collectd_interval: 10
 collectd_readthreads: 7
 collectd_hostname: "{{ inventory_hostname }}"
-collectd_fdqnlookup: false
+collectd_fdqnlookup: true
 
 # Collectd plugins
 collectd_plugins: []                # Ex. [nginx, memcached]
@@ -35,6 +42,12 @@ collectd_default_plugins_options:
   interface:
   - Interface lo
   - IgnoreSelected true
+  cpu:
+  - ValuesPercentage true
+  - ReportByState false
+  - ReportByCpu true
+  - ReportNumCpu true
+
 
 # Additional types
 # format: { name: ..., value: ... }
@@ -55,7 +68,7 @@ collectd_write_graphite_options:    # Setup write_graphite (https://collectd.org
 
 # Setup logs
 collectd_logpath:                   # If it is not empty, will be used logfile
-collectd_loglevel: info
+collectd_loglevel: notice
 collectd_logrotate: yes
 collectd_type: yes
 collectd_logrotate_options:
@@ -66,10 +79,56 @@ collectd_logrotate_options:
   - rotate 7
   - size 10M
 
+# If the csv path is set the plugin will be enabled
+#'/var/lib/collectd/csv'
+collectd_csv_logdir:
+collectd_csv_logdir_storerates: true
+
 # Paths
 collectd_types_db_path: "{{ '/usr/share/collectd/types.db' if collectd_use_ppa else collectd_prefix + '/share/collectd/types.db' }}"
 collectd_additional_types_db_path: "{{ '/usr/share/collectd/additional-types.db' if collectd_use_ppa else collectd_prefix_type + '/types.db' }}"
 collectd_config_file: "{{ '/etc/collectd/collectd.conf' if collectd_use_ppa else collectd_prefix_conf + '/collectd.conf' }}"
 collectd_config_dir: "{{ '/etc/collectd/collectd.conf.d' if collectd_use_ppa else collectd_plugins_prefix }}"
 collectd_plugins_dir: "{{ '/usr/lib/collectd' if collectd_use_ppa else collectd_prefix + '/lib/collectd' }}"
 collectd_sbin_path: "{{ '/usr/sbin' if collectd_use_ppa else collectd_prefix_sbin }}"
+
+collectd_write_amqp: yes
+collectd_write_amqp_options:
+  Host: "indices-manager"
+  Port: "5672"
+  User: "indices_user"
+  Password: "indices_manager"
+  Exchange: "collectd-exchange"
+  RoutingKey: "indices-perf-key"
+
+
+
+collectd_numa_cpu_aggr: yes
+collectd_numa_node_cpus: [ "0 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38",
+                         "1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31 33 35 37 39"]
+
+collectd_numa_cpu_aggr_options:
+   CalculateAverage: true
+   CalculateSum: false
+
+
+# Enable docker monitoring by default
+docker_stats: yes
+
+
+
+# Enable docker monitoring by default
+collectd_intel_pmutools: no
+pmutools_event_dir: ""
+pmu_tools_dir: "/opt/pmu-tools"
+collectd_intel_pmutools_options:
+  ReportHardwareCacheEvents: true
+  ReportKernelPMUEvents: true
+  ReportSoftwareEvents: true
+  HardwareEvents: "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS,L2_RQSTS.ALL_$"
+
+collectd_intel_rdt: no
+collectd_intel_rdt_interval: 1
+collectd_intel_rdt_options:
+  Cores: ""
+

inventory

@@ -0,0 +1,2 @@
+[localhost]
+localhost ansible_connection=local

tasks/collectd.yml

@@ -10,13 +10,19 @@
   when: ansible_os_family == 'Debian' and collectd_service_autodetect
 
 - include: install.ppa.yml
-  when: collectd_use_ppa
+  when: install_collectd and collectd_use_ppa
   tags: [collectd, collectd-install]
 
 - include: install.deb.yml
-  when: not collectd_use_ppa and ansible_os_family == 'Debian'
+  when: install_collectd and not collectd_use_ppa and not collectd_use_git and ansible_os_family == 'Debian'
   tags: [collectd, collectd-install]
 
+
+- include: install.git.yml
+  when: install_collectd and not collectd_use_ppa and collectd_use_git and ansible_os_family == 'Debian'
+  tags: [collectd, collectd-install]
+
+
 - include: configure.yml
   tags: [collectd, collectd-configure]
 

tasks/configure.yml

@@ -10,3 +10,64 @@
 - name: collectd-configure | Configure additional types
   template: src=types.db.j2 dest={{collectd_additional_types_db_path}}
   notify: collectd restart
+
+- name: collectd-configure | Configure Addiotnal Plugin AMQP
+  template: src=amqp.conf.j2 dest={{collectd_config_dir}}/amqp.conf validate='{{collectd_sbin_path}}/collectd -t -C %s'
+  notify: collectd restart
+  when: collectd_write_amqp
+
+- name: collectd-configure | Configure Additional Plugin NUMA CPU Aggregation
+  template: src=numa_cpu_aggregation.conf.j2 dest={{collectd_config_dir}}/numa_cpu_aggregation.conf validate='{{collectd_sbin_path}}/collectd -t -C %s'
+  notify: collectd restart
+  when: collectd_numa_cpu_aggr
+
+- name: collectd-configure | Configure Additional Plugin Docker Stats 1 of 3
+  git: 
+    dest: /usr/share/collectd/docker-collectd-plugin
+    #repo: "https://github.com/lebauce/docker-collectd-plugin.git"
+    #repo: "https://github.com/signalfx/docker-collectd-plugin.git"
+    repo: "https://github.com/doc-vu/docker-collectd-plugin.git"
+    force: yes
+  when: docker_stats
+
+- name: collectd-configure | Configure Additional Plugin Docker Stats 2 of 3
+  shell: pip3 install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt && pip install -r /usr/share/collectd/docker-collectd-plugin/requirements.txt
+  # notify: collectd restart
+  when: docker_stats
+
+- name: collectd-configure | Configure Additional Plugin Docker Stats 3 of 3
+  template: src=docker.conf.j2 dest={{collectd_config_dir}}/docker.conf validate='{{collectd_sbin_path}}/collectd -t -C %s'
+  notify: collectd restart
+  when: docker_stats
+
+- name: collectd-configure | Configure Additional Plugin intel_pmutools 1 of 4
+  shell: cat "{{ pmu_tools_dir }}"/eventlist_info.txt
+  become: yes
+  register: event_dir
+  when: collectd_intel_pmutools
+
+- name: collectd-configure | Configure Additional Plugin intel_pmutools 2 of 4
+  set_fact: 
+    pmutools_event_dir: "{{ event_dir.stdout|default([])  }}"
+  when: collectd_intel_pmutools
+
+- name: collectd-configure | Configure Additional Plugin intel_pmutools 3 of 4
+  debug:
+    var: pmutools_event_dir
+    verbosity: 4
+  when: collectd_intel_pmutools
+
+# - name: collectd-configure | Configure Additional Plugin intel_pmutools 4 of 5
+#   var: pmutools_event_dir
+#   when: collectd_intel_pmutools
+
+- name: collectd-configure | Configure Additional Plugin intel_pmutools 4 of 4
+  template: src=intelpmutool.conf.j2 dest={{collectd_config_dir}}/intelpmutool.conf validate='{{collectd_sbin_path}}/collectd -t -C %s'
+  notify: collectd restart
+  when: collectd_intel_pmutools
+
+- name: collectd-configure | Configure Additional Plugin intel_rdt
+  template: src=intelrdt.conf.j2 dest={{collectd_config_dir}}/intelrdt.conf validate='{{collectd_sbin_path}}/collectd -t -C %s'
+  notify: collectd restart
+  when: collectd_intel_rdt
+

tasks/install.git.yml

@@ -0,0 +1,41 @@
+---
+
+- name: collectd-install | Install dependencies
+  apt: name={{item}} force=yes
+  with_items: [build-essential, libcurl4-openssl-dev, libpcap-dev, libyajl-dev, flex, bison, pkg-config, automake]
+
+- name: collectd-install | Download Collectd
+  git:
+    dest: "{{ collectd_prefix_git_src }}"
+    repo: "https://github.com/collectd/collectd.git"
+  become: yes
+  register: collectd_source
+
+- name: collectd-install | Install Collectd
+  shell: ./build.sh && ./configure --prefix={{collectd_prefix}} --with-python LIBPYTHON_CPPFLAGS=/usr/include/python2.7 LIBPYTHON_LDFLAGS=/usr/local/lib/python2.7 PYTHON_CONFIG=/usr/bin/python2.7-config &&  make -j16 && make all install -j16
+  args:
+    chdir: "{{ collectd_prefix_git_src }}"
+  # when: collectd_source.changed
+  become: yes
+  ignore_error: yes
+
+- name: collectd-install | Create /etc/init.d/collectd
+  template: src=initd.collectd.j2 dest=/etc/init.d/collectd
+  become: yes
+
+- name: collectd-install | create run/collectd.pid file  
+  shell: mkdir -p "{{collectd_prefix}}/run"
+  becom: yes
+
+
+- name: collectd-install | make it executable /etc/init.d/collectd
+  shell: chmod +x /etc/init.d/collectd
+  become: yes
+
+- name: collectd-install | Enable collectd.service
+  shell: systemctl enable collectd.service
+  become: yes
+
+- name: collectd-install | start the collectd.service 
+  service: name=collectd state=started enabled=yes
+  become: yes

tasks/install.ppa.yml

@@ -4,12 +4,33 @@
   apt_repository: >
     repo={{ collectd_ppa_source }}
     update_cache=yes
+  when: not collectd_use_ppa_ci 
+
+- name: collectd-install | add signing key
+  apt_key:
+    url: "http://pkg.ci.collectd.org/pubkey.asc"
+    state: present
+  when: collectd_use_ppa_ci 
 
+- name: collectd-install | add ppa key 
+  apt_repository:
+    repo: deb http://pkg.ci.collectd.org/deb/ {{ansible_distribution_release}} {{collectd_use_ppa_ci_codename}}
+    state: present
+    filename: collectd-ci.list
+    update_cache: yes
+  when: collectd_use_ppa_ci 
+
+# TODO: refine this to decide only based on recommended packages or not
 - name: collectd-install | Install specific collectd package
   apt: >
     name=collectd={{ collectd_version }}
   when: not collectd_use_ppa_latest
 
+- name: collectd-install | Install latest collectd minimal package
+  apt: name=collectd
+       install_recommends=no
+  when: collectd_use_ppa_latest and collectd_use_ppa_minimal
+
 - name: collectd-install | Install latest collectd package
   apt: name=collectd
-  when: collectd_use_ppa_latest
+  when: collectd_use_ppa_latest and not collectd_use_ppa_minimal

templates/amqp.conf.j2

@@ -0,0 +1,12 @@
+LoadPlugin amqp
+<Plugin amqp>
+    <Publish "name-amqp">
+    {% for name, value in collectd_write_amqp_options.items() %}
+    {{ name }} "{{ value }}"
+    {% endfor %}
+    VHost "/"
+    Persistent false
+    StoreRates true
+    Format "json"
+    </Publish>
+</Plugin>

templates/collectd.conf.j2

@@ -17,6 +17,7 @@ LoadPlugin logfile
         LogLevel "{{collectd_loglevel}}"
         File "{{collectd_logpath}}"
         Timestamp true
+        PrintSeverity true
 </Plugin>
 {% else %}
 LoadPlugin syslog
@@ -26,6 +27,18 @@ LoadPlugin syslog
 </Plugin>
 {% endif %}
 
+# Logging CSV
+#StoreRates true|false
+#If set to true, convert counter values to rates. If set to false (the default) counter values are stored as is, i. e. as an increasing integer number.
+{% if collectd_csv_logdir %}
+LoadPlugin csv
+<Plugin "csv">
+  DataDir "{{collectd_csv_logdir}}"
+  StoreRates "{{collectd_csv_logdir_storerates|default('true')}}"
+</Plugin>
+{% endif %}
+
+
 # Default plugins
 {% for plugin in collectd_default_plugins %}
 LoadPlugin {{ plugin }}

templates/docker.conf.j2

@@ -0,0 +1,12 @@
+{% if docker_stats %}
+TypesDB "/usr/share/collectd/docker-collectd-plugin/dockerplugin.db"
+LoadPlugin python
+<Plugin python>
+  ModulePath "/usr/share/collectd/docker-collectd-plugin"
+  Import "dockerplugin"
+  <Module dockerplugin>
+    BaseURL "unix://var/run/docker.sock"
+    Timeout 3
+  </Module>
+</Plugin>
+{% endif %}