Merge pull request #240 from hubblestack/develop

Merge to master (prep v2.2.10)

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.2.9'
+__version__ = '2.2.10'

hubblestack/extmods/modules/pulsar.py

@@ -18,6 +18,7 @@
 import os
 import re
 import yaml
+from salt.exceptions import CommandExecutionError
 
 # Import salt libs
 import salt.ext.six

hubblestack/extmods/modules/win_pulsar.py

@@ -234,7 +234,7 @@ def canary(change_file=None):
         conf_dir = os.path.dirname(__opts__['conf_file'])
         change_file = os.path.join(conf_dir, 'fim_canary.tmp')
     __salt__['file.touch'](change_file)
-    __salt__['file.remove'](change_file)
+    os.remove(change_file)
 
 
 def _check_acl(path, mask, wtype, recurse):
@@ -245,6 +245,7 @@ def _check_acl(path, mask, wtype, recurse):
     else:
         wtype = [wtype]
 
+    path = "'" + path + "'"
     audit_acl = __salt__['cmd.run']('(Get-Acl {0} -Audit).Audit | fl'.format(path), shell='powershell',
                                     python_shell=True)
     if not audit_acl:

hubblestack/files/hubblestack_nova/win_pkg.py

@@ -14,6 +14,7 @@
 import logging
 import salt.utils
 from salt.exceptions import CommandExecutionError
+from distutils.version import LooseVersion
 
 
 log = logging.getLogger(__name__)
@@ -66,7 +67,7 @@ def audit(data_list, tags, debug=False, **kwargs):
                 # Whitelisted audit (must include)
                 if 'whitelist' in audit_type:
                     if name in __pkgdata__:
-                        audit_value = __pkgdata__['name']
+                        audit_value = __pkgdata__[name]
                         tag_data['found_value'] = audit_value
                         secret = _translate_value_type(audit_value, tag_data['value_type'], match_output)
                         if secret:
@@ -152,7 +153,10 @@ def _get_tags(data):
 
 
 def _translate_value_type(current, value, evaluator):
-    if int(current) >= int(evaluator):
+    if 'equal' in value.lower() and LooseVersion(current) == LooseVersion(evaluator):
         return True
-    else:
-        return False
+    if 'less' in value.lower() and LooseVersion(current) <= LooseVersion(evaluator):
+        return True 
+    if 'more' in value.lower() and LooseVersion(current) >= LooseVersion(evaluator):
+        return True 
+    return False

hubblestack/files/hubblestack_nova/win_reg.py

@@ -67,7 +67,7 @@ def audit(data_list, tags, debug=False, **kwargs):
                     current = _find_option_value_in_reg(reg_dict['hive'], reg_dict['key'], reg_dict['value'])
                     if isinstance(current, dict):
                         tag_data['value_found'] = current
-                        if False in current.values():
+                        if any( x is False for x in current.values()):
                             ret['Failure'].append(tag_data)
                         else:
                             answer_list = []
@@ -195,8 +195,8 @@ def _find_option_value_in_reg(reg_hive, reg_key, reg_value):
         for sid in key_list:
             if len(sid) <= 15 or '_Classes' in sid:
                 continue
-            reg_key = reg_key.replace('<SID>', sid)
-            reg_result = __salt__['reg.read_value'](reg_hive, reg_key, reg_value)
+            temp_reg_key = reg_key.replace('<SID>', sid)
+            reg_result = __salt__['reg.read_value'](reg_hive, temp_reg_key, reg_value)
             if reg_result['success']:
                 if reg_result['vdata'] == '(value not set)':
                     ret_dict[sid] = False

pkg/amazonlinux2016.09/Dockerfile

@@ -16,7 +16,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN yum -y install git make python ruby sudo which
@@ -85,14 +85,14 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \
  && pip -v install -r pyinstaller-requirements.txt
 
 #fpm package making requirements start
-RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \
+RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
  && gem install --no-ri --no-rdoc fpm
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/amazonlinux2017.03/Dockerfile

@@ -16,7 +16,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN yum -y install git make python ruby sudo which
@@ -85,14 +85,14 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \
  && pip -v install -r pyinstaller-requirements.txt
 
 #fpm package making requirements start
-RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \
+RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
  && gem install --no-ri --no-rdoc fpm
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/centos6/Dockerfile

@@ -16,7 +16,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN yum -y install git make python ruby sudo which python-argparse
@@ -87,14 +87,14 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \
  && scl enable python27 "pip -v install -r pyinstaller-requirements.txt"
 
 #fpm package making requirements start
-RUN yum install -y rpmbuild gcc make rh-ruby23 rh-ruby23-ruby-devel \
+RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
  && scl enable rh-ruby23 "gem install --no-ri --no-rdoc fpm"
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/centos7/Dockerfile

@@ -16,7 +16,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN yum -y install git make python ruby sudo which
@@ -84,14 +84,14 @@ RUN wget -c https://bootstrap.pypa.io/get-pip.py \
  && pip -v install -r pyinstaller-requirements.txt
 
 #fpm package making requirements start
-RUN yum install -y ruby ruby-devel rpmbuild rubygems gcc make \
+RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
  && gem install --no-ri --no-rdoc fpm
 
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/coreos/Dockerfile

@@ -17,7 +17,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN apt-get -y install git make python ruby sudo
@@ -88,8 +88,8 @@ RUN pip install --upgrade pip \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian7/Dockerfile

@@ -115,8 +115,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian8/Dockerfile

@@ -17,7 +17,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN apt-get -y install git make python ruby sudo locales
@@ -96,8 +96,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian9/Dockerfile

@@ -17,7 +17,7 @@ RUN mkdir -p /etc/osquery /var/log/osquery /etc/hubble/hubble.d /opt/hubble /opt
 #osquery should be built first since requirements for other packages can interfere with osquery dependencies
 #to build, osquery scripts want sudo and a user to sudo with.
 #to pin to a different version change the following envirnment variable
-ENV OSQUERY_SRC_VERSION=2.8.0
+ENV OSQUERY_SRC_VERSION=2.9.2
 ENV OSQUERY_BUILD_USER=osquerybuilder
 ENV OSQUERY_GIT_URL=https://github.com/facebook/osquery.git
 RUN apt-get -y install git make python ruby sudo
@@ -92,8 +92,8 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.2.9
-ENV HUBBLE_VERSION=2.2.9
+ENV HUBBLE_CHECKOUT=v2.2.10
+ENV HUBBLE_VERSION=2.2.10
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

test-requirements.txt

@@ -0,0 +1,7 @@
+#
+############# test-requirements.txt ####################
+#
+pytest
+pyyaml
+salt-ssh
+#

tests/README.md

@@ -0,0 +1,20 @@
+# Hubble Unittests
+
+## Running unit tests
+
+Follow the steps to run unit tests for Hubble
+```
+yum install git wget vim python python-setuptools -y
+easy_install pip
+git clone https://github.com/hubblestack/hubble.git
+cd hubble
+pip install -r test-requirements.txt
+py.test (this will run all the test files in /tests/unittests/)
+py.test tests/unittests/test_pulsar.py (this will run the test file mentioned)
+```
+
+## Adding new unit test
+
+The files in `/tests/unittests/` are unit tests. We are using pytest framework to write unit tests. If you want to add new tests please use the same framework. The new unit tests can be added at the path `/tests/unittests/`.
+
+[Python Unit Testing](https://wiki.corp.adobe.com/display/CoreServicesTeam/Python+Unit+Testing) CST wiki is helpful to understand pytest framework and to write new unit tests.

tests/unittests/resources/top.nebula

@@ -0,0 +1,3 @@
+nebula:
+  '*':
+    - hubblestack_nebula_queries

tests/unittests/resources/top.pulsar

@@ -0,0 +1,3 @@
+pulsar:
+  '*':
+    - hubblestack_pulsar_config