Merge pull request #26 from digitalreasoning/sumologic

Sumologic

SPECFILE

@@ -19,6 +19,10 @@ components:
     sls_path: dr.yourkit
 
 pillar_defaults:
+  sumologic:
+    install: false
+    accessid:
+    accesskey:
   dr:
     aws_access_key: MUST_PROVIDE_VALUE
     aws_secret_key: MUST_PROVIDE_VALUE

dr/base/init.sls

@@ -1,4 +1,6 @@
-
+include:
+  - dr.base.sumologic
+
 # turn off iptables
 {% for svc in [ "iptables", "ip6tables"] %}
 disable-{{svc}}:

dr/base/scripts/installsumo.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+wget https://collectors.sumologic.com/rest/download/linux/64 -O /tmp/sumocollector.sh
+chmod +x /tmp/sumocollector.sh
+/tmp/sumocollector.sh -q -Vsumo.accessid={{ pillar.sumologic.accessid }} -Vsumo.accesskey={{ pillar.sumologic.accesskey }} -Vcollector.name={{ grains.id }} -VsyncSources=/opt/SumoCollector/sources -Vephemeral=true

dr/base/sources/datanode.json

@@ -0,0 +1,32 @@
+{
+  "api.version":"v1",
+  "sources":[
+    {
+      "sourceType":"LocalFile",
+      "name":"HDFS Log",
+      "pathExpression":"/var/log/hadoop-hdfs/hadoop-hdfs-*.log"
+    },
+    {
+      "sourceType":"LocalFile",
+      "name":"HDFS Out",
+      "pathExpression":"/var/log/hadoop-hdfs/hadoop-hdfs-*.out"
+    },
+    {
+      "sourceType":"LocalFile",
+      "name":"YARN Log",
+      "pathExpression":"/var/log/hadoop-yarn/yarn-yarn-nodemanager*.log"
+    },
+    {
+      "sourceType":"LocalFile",
+      "name":"YARN Out",
+      "pathExpression":"/var/log/hadoop-yarn/yarn-yarn-nodemanager*.out"
+    },
+    {
+      "sourceType":"SystemStats",
+      "name":"Disk Used",
+      "interval":300000,
+      "hostName": "{{ grains.id }}",
+      "metrics": ["Disk_Used"]
+    }
+  ]
+}

dr/base/sources/edgenode.json

@@ -0,0 +1,17 @@
+{
+  "api.version":"v1",
+  "sources":[
+    {
+      "sourceType":"LocalFile",
+      "name":"EdgeNode Log",
+      "pathExpression":"/mnt/grater/*/synthesys/synthesys-qa*/logs/*"
+    },
+    {
+      "sourceType":"SystemStats",
+      "name":"Disk Used",
+      "interval":300000,
+      "hostName": "{{ grains.id }}",
+      "metrics": ["Disk_Used"]
+    }
+  ]
+}

dr/base/sources/esdata.json

@@ -0,0 +1,8 @@
+{
+  "api.version":"v1",
+  "source":{
+     "sourceType":"LocalFile",
+     "name":"ES Log",
+     "pathExpression":"/mnt/elasticsearch/logs/*"
+  }
+}

dr/base/sources/oozie.json

@@ -0,0 +1,8 @@
+{
+  "api.version":"v1",
+  "source":{
+     "sourceType":"LocalFile",
+     "name":"Var Log",
+     "pathExpression":"/var/log/oozie*"
+  }
+}

dr/base/sources/yarn.json

@@ -0,0 +1,20 @@
+{
+  "api.version":"v1",
+  "sources":[
+    {
+      "sourceType":"LocalFile",
+      "name":"YARN syslog",
+      "pathExpression":"/mnt/hadoop/yarn/logs/**/syslog"
+    },
+    {
+      "sourceType":"LocalFile",
+      "name":"YARN stderr",
+      "pathExpression":"/mnt/hadoop/yarn/logs/**/stderr"
+    },
+    {
+      "sourceType":"LocalFile",
+      "name":"YARN stdout",
+      "pathExpression":"/mnt/hadoop/yarn/logs/**/stdout"
+    }
+  ]
+}

dr/base/sumologic.sls

@@ -0,0 +1,109 @@
+{% if pillar.sumologic.install %}
+
+# Get roles of this host
+{% set datanode = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.hadoop.hdfs.datanode', 'grains.items', 'compound') %}
+{% set yarn = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.hadoop.yarn.nodemanager', 'grains.items', 'compound') %}
+{% set esdata = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:elasticsearch.data', 'grains.items', 'compound') %}
+{% set edgenode = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.hadoop.client', 'grains.items', 'compound') %}
+{% set oozie = salt['mine.get']('G@stack_id:' ~ grains.stack_id ~ ' and G@roles:cdh5.oozie', 'grains.items', 'compound') %}
+
+{% set anyroles = datanode or yarn or esdata or edgenode or oozie %}
+{% if anyroles %}
+collector_sources:
+  file.directory:
+    - name: /opt/SumoCollector/sources
+    - user: root
+    - group: root
+    - mode: 755
+    - makedirs: true
+{% endif %}
+
+# Install log sources for this host
+{% if datanode %}
+/opt/SumoCollector/sources/datanode.json:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+    - template: jinja
+    - source: salt://dr/base/sources/datanode.json
+    - requires:
+      - file: collector_sources
+{% endif %}
+
+{% if yarn %}
+/opt/SumoCollector/sources/yarn.json:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+    - source: salt://dr/base/sources/yarn.json
+    - requires:
+      - file: collector_sources
+{% endif %}
+
+{% if esdata %}
+/opt/SumoCollector/sources/esdata.json:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+    - source: salt://dr/base/sources/esdata.json
+    - requires:
+      - file: collector_sources
+{% endif %}
+
+{% if edgenode %}
+/opt/SumoCollector/sources/edgenode.json:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+    - source: salt://dr/base/sources/edgenode.json
+    - requires:
+      - file: collector_sources
+{% endif %}
+
+{% if oozie %}
+/opt/SumoCollector/sources/oozie.json:
+  file.managed:
+    - user: root
+    - group: root
+    - mode: 644
+    - source: salt://dr/base/sources/oozie.json
+    - requires:
+      - file: collector_sources
+{% endif %}
+
+
+{% if anyroles %}
+# Install the package
+wget:
+  pkg.installed
+
+script:
+  file.managed:
+    - name: /tmp/installsumo.sh
+    - source: salt://dr/base/scripts/installsumo.sh
+    - user: root
+    - group: root
+    - mode: 777
+    - template: jinja
+    - requires:
+      - pkg: wget
+
+install:
+  cmd.run:
+    - name: /tmp/installsumo.sh
+    - requires:
+      - file: script
+
+# Start the collector
+collector:
+  service.running:
+    - enable: True
+    - watch:
+      - cmd: install
+{% endif %}
+
+{% endif %}