Merge pull request #33 from hubblestack/develop

Merge to master (prep v2.1.3 release)

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.1.2'
+__version__ = '2.1.3'

hubblestack/extmods/modules/nebula_osquery.py

@@ -99,6 +99,15 @@ def queries(query_group,
                              'result': True
                         }}
                 )
+            uptime = __salt__['status.uptime']()
+            if isinstance(uptime, dict):
+                uptime = uptime.get('seconds', __salt__['cmd.run']('uptime'))
+            ret.append(
+                    {'fallback_uptime': {
+                         'data': [{'uptime': uptime}],
+                         'result': True
+                    }}
+            )
             if report_version_with_day:
                 ret.append(hubble_versions())
             return ret

hubblestack/extmods/returners/splunk_nebula_return.py

@@ -52,7 +52,7 @@
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
-http_event_collector_debug = True
+http_event_collector_debug = False
 
 log = logging.getLogger(__name__)
 
@@ -70,6 +70,19 @@ def returner(ret):
     proxy = opts['proxy']
     timeout = opts['timeout']
     custom_fields = opts['custom_fields']
+
+    # Gather amazon information if present
+    aws_ami_id = None
+    aws_instance_id = None
+    try:
+        aws_ami_id = requests.get('http://169.254.169.254/latest/meta-data/ami-id',
+                                  timeout=1).text
+        aws_instance_id = requests.get('http://169.254.169.254/latest/meta-data/instance-id',
+                                       timeout=1).text
+    except requests.exceptions.ConnectTimeout:
+        # Not on an AWS box
+        pass
+
     # Set up the collector
     hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
 
@@ -79,6 +92,8 @@ def returner(ret):
     jid = ret['jid']
     master = __grains__['master']
     fqdn = __grains__['fqdn']
+    # Sometimes fqdn is blank. If it is, replace it with minion_id
+    fqdn = fqdn if fqdn else minion_id
     try:
         fqdn_ip4 = __grains__['fqdn_ip4'][0]
     except IndexError:
@@ -100,6 +115,10 @@ def returner(ret):
                     event.update({'dest_host': fqdn})
                     event.update({'dest_ip': fqdn_ip4})
 
+                    if aws_instance_id is not None:
+                        event.update({'aws_ami_id': aws_ami_id})
+                        event.update({'aws_instance_id': aws_instance_id})
+
                     for custom_field in custom_fields:
                         custom_field_name = 'custom_' + custom_field
                         custom_field_value = __salt__['config.get'](custom_field, '')

hubblestack/extmods/returners/splunk_nova_return.py

@@ -52,7 +52,7 @@
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
-http_event_collector_debug = True
+http_event_collector_debug = False
 
 log = logging.getLogger(__name__)
 
@@ -69,13 +69,28 @@ def returner(ret):
     proxy = opts['proxy']
     timeout = opts['timeout']
     custom_fields = opts['custom_fields']
+
+    # Gather amazon information if present
+    aws_ami_id = None
+    aws_instance_id = None
+    try:
+        aws_ami_id = requests.get('http://169.254.169.254/latest/meta-data/ami-id',
+                                  timeout=1).text
+        aws_instance_id = requests.get('http://169.254.169.254/latest/meta-data/instance-id',
+                                       timeout=1).text
+    except requests.exceptions.ConnectTimeout:
+        # Not on an AWS box
+        pass
+
     # Set up the collector
     hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
     # st = 'salt:hubble:nova'
     data = ret['return']
     minion_id = ret['id']
     jid = ret['jid']
     fqdn = __grains__['fqdn']
+    # Sometimes fqdn is blank. If it is, replace it with minion_id
+    fqdn = fqdn if fqdn else minion_id
     master = __grains__['master']
     try:
         fqdn_ip4 = __grains__['fqdn_ip4'][0]
@@ -105,6 +120,10 @@ def returner(ret):
         event.update({'dest_host': fqdn})
         event.update({'dest_ip': fqdn_ip4})
 
+        if aws_instance_id is not None:
+            event.update({'aws_ami_id': aws_ami_id})
+            event.update({'aws_instance_id': aws_instance_id})
+
         for custom_field in custom_fields:
             custom_field_name = 'custom_' + custom_field
             custom_field_value = __salt__['config.get'](custom_field, '')
@@ -138,6 +157,10 @@ def returner(ret):
         event.update({'dest_host': fqdn})
         event.update({'dest_ip': fqdn_ip4})
 
+        if aws_instance_id is not None:
+            event.update({'aws_ami_id': aws_ami_id})
+            event.update({'aws_instance_id': aws_instance_id})
+
         for custom_field in custom_fields:
             custom_field_name = 'custom_' + custom_field
             custom_field_value = __salt__['config.get'](custom_field, '')
@@ -163,6 +186,10 @@ def returner(ret):
         event.update({'dest_host': fqdn})
         event.update({'dest_ip': fqdn_ip4})
 
+        if aws_instance_id is not None:
+            event.update({'aws_ami_id': aws_ami_id})
+            event.update({'aws_instance_id': aws_instance_id})
+
         for custom_field in custom_fields:
             custom_field_name = 'custom_' + custom_field
             custom_field_value = __salt__['config.get'](custom_field, '')

hubblestack/extmods/returners/splunk_pulsar_return.py

@@ -75,6 +75,19 @@ def returner(ret):
     proxy = opts['proxy']
     timeout = opts['timeout']
     custom_fields = opts['custom_fields']
+
+    # Gather amazon information if present
+    aws_ami_id = None
+    aws_instance_id = None
+    try:
+        aws_ami_id = requests.get('http://169.254.169.254/latest/meta-data/ami-id',
+                                  timeout=1).text
+        aws_instance_id = requests.get('http://169.254.169.254/latest/meta-data/instance-id',
+                                       timeout=1).text
+    except requests.exceptions.ConnectTimeout:
+        # Not on an AWS box
+        pass
+
     # Set up the collector
     hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout)
     # Check whether or not data is batched:
@@ -86,6 +99,8 @@ def returner(ret):
     data = _dedupList(data)
     minion_id = __opts__['id']
     fqdn = __grains__['fqdn']
+    # Sometimes fqdn is blank. If it is, replace it with minion_id
+    fqdn = fqdn if fqdn else minion_id
     master = __grains__['master']
     try:
         fqdn_ip4 = __grains__['fqdn_ip4'][0]
@@ -193,6 +208,10 @@ def returner(ret):
         event.update({'dest_host': fqdn})
         event.update({'dest_ip': fqdn_ip4})
 
+        if aws_instance_id is not None:
+            event.update({'aws_ami_id': aws_ami_id})
+            event.update({'aws_instance_id': aws_instance_id})
+
         for custom_field in custom_fields:
             custom_field_name = 'custom_' + custom_field
             custom_field_value = __salt__['config.get'](custom_field, '')

hubblestack/files/hubblestack_nova/cve_scan_v2.py

@@ -168,7 +168,7 @@ def audit(data_list, tags, verbose=False, show_profile=False, debug=False):
                         os.remove(cached_zip)
                         extracted_json = os.path.join(__opts__['cachedir'],
                                                       'cve_scan_cache',
-                                                      '%s_%s.json' % (os_name, os_version.replace('.', '')))
+                                                      '%s_%s.json' % (os_name, str(os_version).replace('.', '')))
                         log.debug('attempting to open %s', extracted_json)
                         with open(extracted_json, 'r') as json_file:
                             master_json = json.load(json_file)

pkg/build_debs.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -x # echo on
+
+_user=`id -u`
+
+# Check if the current user is root
+if [ "$_user" == "0" ]
+then
+  echo "This script should not be run as root ..."
+  echo "Please run this script as regular user with sudo privileges ..."
+  echo "Exiting ..."
+  exit
+fi
+
+cd ..
+
+bash init_pkg.sh -y
+cp hubble.tar.gz ~/hubble.tar.gz
+rm -rf ~/hubblestack-2.1.3
+rm -rf ~/hubblestack-2.1.3.tar.gz
+mkdir ~/hubblestack-2.1.3
+tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.3
+mkdir -p ~/hubblestack-2.1.3/etc/init.d
+cp pkg/hubble ~/hubblestack-2.1.3/etc/init.d
+mkdir -p ~/hubblestack-2.1.3/usr/lib/systemd/system
+cp pkg/hubble.service ~/hubblestack-2.1.3/usr/lib/systemd/system
+cp -f conf/hubble ~/hubblestack-2.1.3/etc/hubble/hubble
+cd ~/hubblestack-2.1.3
+
+sudo apt-get install -y ruby ruby-dev rubygems gcc make
+sudo gem install --no-ri --no-rdoc fpm
+mkdir -p usr/bin
+ln -s /opt/hubble/hubble usr/bin/hubble
+ln -s /opt/osquery/osqueryd usr/bin/osqueryd
+ln -s /opt/osquery/osqueryi usr/bin/osqueryi
+fpm -s dir -t deb \
+    -n hubblestack \
+    -v 2.1.3-1 \
+    -d 'git' \
+    --config-files /etc/hubble/hubble --config-files /etc/osquery/osquery.conf \
+    --deb-no-default-config-files \
+    etc/hubble etc/osquery etc/init.d opt usr/bin
+cp hubblestack_2.1.3-1_amd64.deb ~/

pkg/build_rpms.sh

@@ -17,17 +17,17 @@ cd ..
 
 bash init_pkg.sh -y
 cp hubble.tar.gz ~/hubble.tar.gz
-rm -rf ~/hubblestack-2.1.2
-rm -rf ~/hubblestack-2.1.2.tar.gz
-mkdir ~/hubblestack-2.1.2
-tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.2
-mkdir -p ~/hubblestack-2.1.2/etc/init.d
-cp pkg/hubble ~/hubblestack-2.1.2/etc/init.d
-mkdir -p ~/hubblestack-2.1.2/usr/lib/systemd/system
-cp pkg/hubble.service ~/hubblestack-2.1.2/usr/lib/systemd/system
-cp -f conf/hubble ~/hubblestack-2.1.2/etc/hubble/hubble
+rm -rf ~/hubblestack-2.1.3
+rm -rf ~/hubblestack-2.1.3.tar.gz
+mkdir ~/hubblestack-2.1.3
+tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.3
+mkdir -p ~/hubblestack-2.1.3/etc/init.d
+cp pkg/hubble ~/hubblestack-2.1.3/etc/init.d
+mkdir -p ~/hubblestack-2.1.3/usr/lib/systemd/system
+cp pkg/hubble.service ~/hubblestack-2.1.3/usr/lib/systemd/system
+cp -f conf/hubble ~/hubblestack-2.1.3/etc/hubble/hubble
 cd ~
-tar -czvf hubblestack-2.1.2.tar.gz hubblestack-2.1.2/
+tar -czvf hubblestack-2.1.3.tar.gz hubblestack-2.1.3/
 rm -rf ~/rpmbuild
 mkdir -p ~/rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp}
 
@@ -36,7 +36,7 @@ cat <<EOF >~/.rpmmacros
 %_tmppath  %{_topdir}/tmp
 EOF
 
-cp ~/hubblestack-2.1.2.tar.gz ~/rpmbuild/SOURCES/
+cp ~/hubblestack-2.1.3.tar.gz ~/rpmbuild/SOURCES/
 cd ~/rpmbuild
 
 cp ~/hubble/pkg/specs/* SPECS/

pkg/specs/hubblestack-el6.spec

@@ -9,7 +9,7 @@
 
 Summary: Hubblestack is a module, open-source security compliance framework
 Name: hubblestack
-Version: 2.1.2
+Version: 2.1.3
 Release: 1
 License: Apache 2.0
 Group: Development/Tools
@@ -53,6 +53,13 @@ rm -rf %{buildroot}
 /usr/bin/*
 
 %changelog
+* Tue Feb 28 2017  Colton Myers <[email protected]> 2.1.3-1
+- Add Debian packaging
+- Add uptime fallback query
+- Fix for blank hosts when fqdn doesn't return anything in returners
+- Add AWS information to events from splunk returners
+- Turn off http_event_collector_debug everywhere
+
 * Mon Feb 13 2017  Colton Myers <[email protected]> 2.1.2-1
 - Fix the changelog order
 

pkg/specs/hubblestack-el7.spec

@@ -9,7 +9,7 @@
 
 Summary: Hubblestack is a module, open-source security compliance framework
 Name: hubblestack
-Version: 2.1.2
+Version: 2.1.3
 Release: 1
 License: Apache 2.0
 Group: Development/Tools
@@ -53,6 +53,13 @@ rm -rf %{buildroot}
 /usr/lib/*
 
 %changelog
+* Tue Feb 28 2017  Colton Myers <[email protected]> 2.1.3-1
+- Add Debian packaging
+- Add uptime fallback query
+- Fix for blank hosts when fqdn doesn't return anything in returners
+- Add AWS information to events from splunk returners
+- Turn off http_event_collector_debug everywhere
+
 * Mon Feb 13 2017  Colton Myers <[email protected]> 2.1.2-1
 - Fix the changelog order