diff --git a/hubblestack/__init__.py b/hubblestack/__init__.py index f81156126..2d31b1c32 100644 --- a/hubblestack/__init__.py +++ b/hubblestack/__init__.py @@ -1 +1 @@ -__version__ = '2.1.2' +__version__ = '2.1.3' diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 9037ff3ea..896153473 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -99,6 +99,15 @@ def queries(query_group, 'result': True }} ) + uptime = __salt__['status.uptime']() + if isinstance(uptime, dict): + uptime = uptime.get('seconds', __salt__['cmd.run']('uptime')) + ret.append( + {'fallback_uptime': { + 'data': [{'uptime': uptime}], + 'result': True + }} + ) if report_version_with_day: ret.append(hubble_versions()) return ret diff --git a/hubblestack/extmods/returners/splunk_nebula_return.py b/hubblestack/extmods/returners/splunk_nebula_return.py index bd6693bd3..f17a0d5ea 100644 --- a/hubblestack/extmods/returners/splunk_nebula_return.py +++ b/hubblestack/extmods/returners/splunk_nebula_return.py @@ -52,7 +52,7 @@ _max_content_bytes = 100000 http_event_collector_SSL_verify = False -http_event_collector_debug = True +http_event_collector_debug = False log = logging.getLogger(__name__) @@ -70,6 +70,19 @@ def returner(ret): proxy = opts['proxy'] timeout = opts['timeout'] custom_fields = opts['custom_fields'] + + # Gather amazon information if present + aws_ami_id = None + aws_instance_id = None + try: + aws_ami_id = requests.get('http://169.254.169.254/latest/meta-data/ami-id', + timeout=1).text + aws_instance_id = requests.get('http://169.254.169.254/latest/meta-data/instance-id', + timeout=1).text + except requests.exceptions.ConnectTimeout: + # Not on an AWS box + pass + # Set up the collector hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout) @@ -79,6 +92,8 @@ def returner(ret): jid = ret['jid'] master = __grains__['master'] fqdn = __grains__['fqdn'] + # Sometimes fqdn is blank. If it is, replace it with minion_id + fqdn = fqdn if fqdn else minion_id try: fqdn_ip4 = __grains__['fqdn_ip4'][0] except IndexError: @@ -100,6 +115,10 @@ def returner(ret): event.update({'dest_host': fqdn}) event.update({'dest_ip': fqdn_ip4}) + if aws_instance_id is not None: + event.update({'aws_ami_id': aws_ami_id}) + event.update({'aws_instance_id': aws_instance_id}) + for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') diff --git a/hubblestack/extmods/returners/splunk_nova_return.py b/hubblestack/extmods/returners/splunk_nova_return.py index 719f0f5c1..131e604be 100644 --- a/hubblestack/extmods/returners/splunk_nova_return.py +++ b/hubblestack/extmods/returners/splunk_nova_return.py @@ -52,7 +52,7 @@ _max_content_bytes = 100000 http_event_collector_SSL_verify = False -http_event_collector_debug = True +http_event_collector_debug = False log = logging.getLogger(__name__) @@ -69,6 +69,19 @@ def returner(ret): proxy = opts['proxy'] timeout = opts['timeout'] custom_fields = opts['custom_fields'] + + # Gather amazon information if present + aws_ami_id = None + aws_instance_id = None + try: + aws_ami_id = requests.get('http://169.254.169.254/latest/meta-data/ami-id', + timeout=1).text + aws_instance_id = requests.get('http://169.254.169.254/latest/meta-data/instance-id', + timeout=1).text + except requests.exceptions.ConnectTimeout: + # Not on an AWS box + pass + # Set up the collector hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout) # st = 'salt:hubble:nova' @@ -76,6 +89,8 @@ def returner(ret): minion_id = ret['id'] jid = ret['jid'] fqdn = __grains__['fqdn'] + # Sometimes fqdn is blank. If it is, replace it with minion_id + fqdn = fqdn if fqdn else minion_id master = __grains__['master'] try: fqdn_ip4 = __grains__['fqdn_ip4'][0] @@ -105,6 +120,10 @@ def returner(ret): event.update({'dest_host': fqdn}) event.update({'dest_ip': fqdn_ip4}) + if aws_instance_id is not None: + event.update({'aws_ami_id': aws_ami_id}) + event.update({'aws_instance_id': aws_instance_id}) + for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') @@ -138,6 +157,10 @@ def returner(ret): event.update({'dest_host': fqdn}) event.update({'dest_ip': fqdn_ip4}) + if aws_instance_id is not None: + event.update({'aws_ami_id': aws_ami_id}) + event.update({'aws_instance_id': aws_instance_id}) + for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') @@ -163,6 +186,10 @@ def returner(ret): event.update({'dest_host': fqdn}) event.update({'dest_ip': fqdn_ip4}) + if aws_instance_id is not None: + event.update({'aws_ami_id': aws_ami_id}) + event.update({'aws_instance_id': aws_instance_id}) + for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') diff --git a/hubblestack/extmods/returners/splunk_pulsar_return.py b/hubblestack/extmods/returners/splunk_pulsar_return.py index f6000b62e..836c0c044 100644 --- a/hubblestack/extmods/returners/splunk_pulsar_return.py +++ b/hubblestack/extmods/returners/splunk_pulsar_return.py @@ -75,6 +75,19 @@ def returner(ret): proxy = opts['proxy'] timeout = opts['timeout'] custom_fields = opts['custom_fields'] + + # Gather amazon information if present + aws_ami_id = None + aws_instance_id = None + try: + aws_ami_id = requests.get('http://169.254.169.254/latest/meta-data/ami-id', + timeout=1).text + aws_instance_id = requests.get('http://169.254.169.254/latest/meta-data/instance-id', + timeout=1).text + except requests.exceptions.ConnectTimeout: + # Not on an AWS box + pass + # Set up the collector hec = http_event_collector(http_event_collector_key, http_event_collector_host, http_event_server_ssl=hec_ssl, proxy=proxy, timeout=timeout) # Check whether or not data is batched: @@ -86,6 +99,8 @@ def returner(ret): data = _dedupList(data) minion_id = __opts__['id'] fqdn = __grains__['fqdn'] + # Sometimes fqdn is blank. If it is, replace it with minion_id + fqdn = fqdn if fqdn else minion_id master = __grains__['master'] try: fqdn_ip4 = __grains__['fqdn_ip4'][0] @@ -193,6 +208,10 @@ def returner(ret): event.update({'dest_host': fqdn}) event.update({'dest_ip': fqdn_ip4}) + if aws_instance_id is not None: + event.update({'aws_ami_id': aws_ami_id}) + event.update({'aws_instance_id': aws_instance_id}) + for custom_field in custom_fields: custom_field_name = 'custom_' + custom_field custom_field_value = __salt__['config.get'](custom_field, '') diff --git a/hubblestack/files/hubblestack_nova/cve_scan_v2.py b/hubblestack/files/hubblestack_nova/cve_scan_v2.py index 1dc83bf14..6e0ac7cb9 100644 --- a/hubblestack/files/hubblestack_nova/cve_scan_v2.py +++ b/hubblestack/files/hubblestack_nova/cve_scan_v2.py @@ -168,7 +168,7 @@ def audit(data_list, tags, verbose=False, show_profile=False, debug=False): os.remove(cached_zip) extracted_json = os.path.join(__opts__['cachedir'], 'cve_scan_cache', - '%s_%s.json' % (os_name, os_version.replace('.', ''))) + '%s_%s.json' % (os_name, str(os_version).replace('.', ''))) log.debug('attempting to open %s', extracted_json) with open(extracted_json, 'r') as json_file: master_json = json.load(json_file) diff --git a/pkg/build_debs.sh b/pkg/build_debs.sh new file mode 100644 index 000000000..5bd91752b --- /dev/null +++ b/pkg/build_debs.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -x # echo on + +_user=`id -u` + +# Check if the current user is root +if [ "$_user" == "0" ] +then + echo "This script should not be run as root ..." + echo "Please run this script as regular user with sudo privileges ..." + echo "Exiting ..." + exit +fi + +cd .. + +bash init_pkg.sh -y +cp hubble.tar.gz ~/hubble.tar.gz +rm -rf ~/hubblestack-2.1.3 +rm -rf ~/hubblestack-2.1.3.tar.gz +mkdir ~/hubblestack-2.1.3 +tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.3 +mkdir -p ~/hubblestack-2.1.3/etc/init.d +cp pkg/hubble ~/hubblestack-2.1.3/etc/init.d +mkdir -p ~/hubblestack-2.1.3/usr/lib/systemd/system +cp pkg/hubble.service ~/hubblestack-2.1.3/usr/lib/systemd/system +cp -f conf/hubble ~/hubblestack-2.1.3/etc/hubble/hubble +cd ~/hubblestack-2.1.3 + +sudo apt-get install -y ruby ruby-dev rubygems gcc make +sudo gem install --no-ri --no-rdoc fpm +mkdir -p usr/bin +ln -s /opt/hubble/hubble usr/bin/hubble +ln -s /opt/osquery/osqueryd usr/bin/osqueryd +ln -s /opt/osquery/osqueryi usr/bin/osqueryi +fpm -s dir -t deb \ + -n hubblestack \ + -v 2.1.3-1 \ + -d 'git' \ + --config-files /etc/hubble/hubble --config-files /etc/osquery/osquery.conf \ + --deb-no-default-config-files \ + etc/hubble etc/osquery etc/init.d opt usr/bin +cp hubblestack_2.1.3-1_amd64.deb ~/ diff --git a/pkg/build_rpms.sh b/pkg/build_rpms.sh index 7d25ecd6c..6c10808f2 100755 --- a/pkg/build_rpms.sh +++ b/pkg/build_rpms.sh @@ -17,17 +17,17 @@ cd .. bash init_pkg.sh -y cp hubble.tar.gz ~/hubble.tar.gz -rm -rf ~/hubblestack-2.1.2 -rm -rf ~/hubblestack-2.1.2.tar.gz -mkdir ~/hubblestack-2.1.2 -tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.2 -mkdir -p ~/hubblestack-2.1.2/etc/init.d -cp pkg/hubble ~/hubblestack-2.1.2/etc/init.d -mkdir -p ~/hubblestack-2.1.2/usr/lib/systemd/system -cp pkg/hubble.service ~/hubblestack-2.1.2/usr/lib/systemd/system -cp -f conf/hubble ~/hubblestack-2.1.2/etc/hubble/hubble +rm -rf ~/hubblestack-2.1.3 +rm -rf ~/hubblestack-2.1.3.tar.gz +mkdir ~/hubblestack-2.1.3 +tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.3 +mkdir -p ~/hubblestack-2.1.3/etc/init.d +cp pkg/hubble ~/hubblestack-2.1.3/etc/init.d +mkdir -p ~/hubblestack-2.1.3/usr/lib/systemd/system +cp pkg/hubble.service ~/hubblestack-2.1.3/usr/lib/systemd/system +cp -f conf/hubble ~/hubblestack-2.1.3/etc/hubble/hubble cd ~ -tar -czvf hubblestack-2.1.2.tar.gz hubblestack-2.1.2/ +tar -czvf hubblestack-2.1.3.tar.gz hubblestack-2.1.3/ rm -rf ~/rpmbuild mkdir -p ~/rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp} @@ -36,7 +36,7 @@ cat <~/.rpmmacros %_tmppath %{_topdir}/tmp EOF -cp ~/hubblestack-2.1.2.tar.gz ~/rpmbuild/SOURCES/ +cp ~/hubblestack-2.1.3.tar.gz ~/rpmbuild/SOURCES/ cd ~/rpmbuild cp ~/hubble/pkg/specs/* SPECS/ diff --git a/pkg/specs/hubblestack-el6.spec b/pkg/specs/hubblestack-el6.spec index 245b3d045..a46e54dad 100644 --- a/pkg/specs/hubblestack-el6.spec +++ b/pkg/specs/hubblestack-el6.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.1.2 +Version: 2.1.3 Release: 1 License: Apache 2.0 Group: Development/Tools @@ -53,6 +53,13 @@ rm -rf %{buildroot} /usr/bin/* %changelog +* Tue Feb 28 2017 Colton Myers 2.1.3-1 +- Add Debian packaging +- Add uptime fallback query +- Fix for blank hosts when fqdn doesn't return anything in returners +- Add AWS information to events from splunk returners +- Turn off http_event_collector_debug everywhere + * Mon Feb 13 2017 Colton Myers 2.1.2-1 - Fix the changelog order diff --git a/pkg/specs/hubblestack-el7.spec b/pkg/specs/hubblestack-el7.spec index 5c77cbedd..18bd49c26 100644 --- a/pkg/specs/hubblestack-el7.spec +++ b/pkg/specs/hubblestack-el7.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.1.2 +Version: 2.1.3 Release: 1 License: Apache 2.0 Group: Development/Tools @@ -53,6 +53,13 @@ rm -rf %{buildroot} /usr/lib/* %changelog +* Tue Feb 28 2017 Colton Myers 2.1.3-1 +- Add Debian packaging +- Add uptime fallback query +- Fix for blank hosts when fqdn doesn't return anything in returners +- Add AWS information to events from splunk returners +- Turn off http_event_collector_debug everywhere + * Mon Feb 13 2017 Colton Myers 2.1.2-1 - Fix the changelog order