Merge pull request #451 from hubblestack/develop

Merge to master (prep v2.4.3)

doc/conf.py

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.4.2'
+version = u'2.4.3'
 # The full version, including alpha/beta/rc tags.
-release = u'2.4.2-1'
+release = u'2.4.3-1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.4.2'
+__version__ = '2.4.3'

hubblestack/extmods/grains/hubbleversion.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+'''
+Add the hubble version to the grains
+'''
+import logging
+
+from hubblestack import __version__
+
+log = logging.getLogger(__name__)
+
+
+def hubble_version():
+    '''
+    Add the hubble version to the grains
+    '''
+    return {'hubble_version': __version__}

hubblestack/extmods/modules/hubble.py

@@ -28,7 +28,10 @@
 import salt.utils
 from salt.exceptions import CommandExecutionError
 from hubblestack import __version__
-from hubblestack.extmods.modules.nova_loader import NovaLazyLoader
+try:
+    from nova_loader import NovaLazyLoader
+except ImportError:
+    pass  # This is here to make the sphinx import of this module work
 
 __nova__ = {}
 

hubblestack/extmods/modules/nebula_osquery.py

@@ -68,13 +68,13 @@ def queries(query_group,
         which was run) will be included in the result.
 
     topfile_for_mask
-        This is the location of the top file from which the masking information 
+        This is the location of the top file from which the masking information
         will be extracted.
-        
+
     mask_passwords
-        Defaults to False. If set to True, passwords mentioned in the 
+        Defaults to False. If set to True, passwords mentioned in the
         return object are masked.
-        
+
     CLI Examples:
 
     .. code-block:: bash
@@ -334,8 +334,8 @@ def mask_passwords_inplace(object_to_be_masked, topfile):
     It masks the passwords present in 'object_to_be_masked'. Uses mask configuration
     file as a reference to find out the list of blacklisted strings or objects.
     Note that this method alters "object_to_be_masked".
-    
-    The path to the mask configuration file can be specified in the "topfile" 
+
+    The path to the mask configuration file can be specified in the "topfile"
     argument.
     '''
     try:
@@ -375,7 +375,7 @@ def mask_passwords_inplace(object_to_be_masked, topfile):
                 for r in object_to_be_masked:
                     for query_result in r.get(query_name, {'data':[]})['data']:
                         if column not in query_result or not isinstance(query_result[column], basestring):
-                            # if the column in not present in one data-object, it will 
+                            # if the column in not present in one data-object, it will
                             # not be present in others as well. Break in that case.
                             # This will happen only if mask.yaml is malformed
                             break
@@ -413,30 +413,30 @@ def mask_passwords_inplace(object_to_be_masked, topfile):
                             (isinstance(query_result[column], basestring) and query_result[column].strip() != '' ):
                                 break
                             _recursively_mask_objects(query_result[column], blacklisted_object, mask_by)
-                                            
+
             # successfully masked the object. No need to return anything
-        
+
     except Exception as e:
         log.exception("An error occured while masking the passwords: {}".format(e))
 
 def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by):
     '''
     This function is used by "mask_passwords_inplace" to mask passwords contained in
-    json objects or json arrays. If the "object_to_mask" is a json array, then this 
+    json objects or json arrays. If the "object_to_mask" is a json array, then this
     function is called recursively on the individual members of the array.
- 
+
      object_to_mask
         Json object/array whose elements are to masked recursively
-        
+
      blacklisted_object
-        This parameters contains info about which queries are to be masked, which 
+        This parameters contains info about which queries are to be masked, which
         attributes are to be masked, based upon the value of which attribute.
         See hubblestack_nebula_v2/mask.yaml for exact format.
-        
+
     mask_by
-        If a password string is detected, it is replaced by the value of "mask_by" 
+        If a password string is detected, it is replaced by the value of "mask_by"
         parameter.
-        
+
     '''
     if isinstance(object_to_mask, list):
         for child in object_to_mask:

hubblestack/splunklogging.py

@@ -187,13 +187,14 @@ def format_record(self, record):
         try:
             log_entry = {'message': record.message,
                          'level': record.levelname,
-                         'timestamp': record.asctime,
+                         'timestamp': int(time.time()),
                          'loggername': record.name,
                          }
         except:
             log_entry = {'message': record.msg,
                          'level': record.levelname,
                          'loggername': record.name,
+                         'timestamp': int(time.time()),
                          }
         return log_entry
 

pkg/amazonlinux2016.09/Dockerfile

@@ -125,8 +125,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src

pkg/centos6/Dockerfile

@@ -127,8 +127,8 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src

pkg/centos7/Dockerfile

@@ -124,8 +124,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src

pkg/coreos/Dockerfile

@@ -136,9 +136,9 @@ RUN pip -v install -r pyinstaller-requirements.txt
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian7/Dockerfile

@@ -163,9 +163,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian8/Dockerfile

@@ -145,9 +145,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian9/Dockerfile

@@ -140,9 +140,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.2
+ENV HUBBLE_CHECKOUT=v2.4.3
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2
+ENV HUBBLE_VERSION=2.4.3
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/amazonlinux2016.09/Dockerfile

@@ -128,7 +128,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/centos6/Dockerfile

@@ -130,7 +130,7 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/centos7/Dockerfile

@@ -127,7 +127,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/coreos/Dockerfile

@@ -139,7 +139,7 @@ RUN pip -v install -r pyinstaller-requirements.txt
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/debian7/Dockerfile

@@ -166,7 +166,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/debian8/Dockerfile

@@ -148,7 +148,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/debian9/Dockerfile

@@ -143,7 +143,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.2_develop
+ENV HUBBLE_VERSION=2.4.3_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"