Merge pull request #457 from hubblestack/develop

Merge to master (prep v2.4.4)

README.rst

@@ -6,3 +6,13 @@ You can find the docs `here <https://docs.hubblestack.io>`_
 You can file an issue `here <https://github.com/hubblestack/hubble/issues/new>`_
 
 Follow us on `Twitter! <https://twitter.com/hubblestack>`_
+
+Development
+===========
+Below are sample instructions to setup a dev environment:
+
+1. virtualenv myvirtualenv
+2. source myvirtualenv/bin/activate
+3. pip install -r requirements.txt
+4. sudo python setup.py develop
+5. sudo hubble hubble.audit

conf/beforeremove.sh

@@ -0,0 +1 @@
+service hubble stop

conf/hubble

@@ -41,7 +41,8 @@ fileserver_backend:
 #  audit_daily:
 #    function: hubble.audit
 #    seconds: 86400
-#    splay: 3600
+#    min_splay: 1800   # due to this min_splay, audit will run 30 to 60 minutes
+#    splay: 3600       # after hubble service starts                    
 #    kwargs:
 #      verbose: True
 #    returner: splunk_nova_return

cp-pyinstaller.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+# NOTE: after running this script check the diff to fix any 
+# unintended changes 
+
+directories=`find pkg -type d | egrep -v "source|scripts|pyinstaller*|dev$|pkg$"`
+
+for i in $directories;
+do
+  cp -f pyinstaller-requirements.txt $i
+done

doc/conf.py

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.4.3'
+version = u'2.4.4'
 # The full version, including alpha/beta/rc tags.
-release = u'2.4.3-1'
+release = u'2.4.4-1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.4.3'
+__version__ = '2.4.4'

hubblestack/daemon.py

@@ -169,8 +169,8 @@ def getsecondsbycronexpression(base, cron_exp):
     this function will return the seconds according to the cron
     expression provided in the hubble config
     '''
-    iter = croniter(cron_exp, base)
-    next_datetime  = iter.get_next(datetime)
+    cron_iter = croniter(cron_exp, base)
+    next_datetime  = cron_iter.get_next(datetime)
     epoch_base_datetime = time.mktime(base.timetuple())
     epoch_datetime = time.mktime(next_datetime.timetuple())
     seconds = int(epoch_datetime) - int(epoch_base_datetime)
@@ -197,8 +197,8 @@ def getlastrunbybuckets(buckets, seconds):
     buckets = int(buckets) if int(buckets)!=0 else 256
     host_ip = socket.gethostbyname(socket.gethostname())
     ips = host_ip.split('.')
-    sum = (int(ips[0])*256*256*256)+(int(ips[1])*256*256)+(int(ips[2])*256)+int(ips[3])
-    bucket = sum%buckets
+    bucket_sum = (int(ips[0])*256*256*256)+(int(ips[1])*256*256)+(int(ips[2])*256)+int(ips[3])
+    bucket = bucket_sum%buckets
     log.debug('bucket number is {0} out of {1}'.format(bucket, buckets))
     current_time = time.time()
     base_time = seconds*(math.floor(current_time/seconds))
@@ -228,6 +228,7 @@ def schedule():
             function: hubble.audit
             seconds: 3600
             splay: 100
+            min_splay: 50
             args:
               - cis.centos-7-level-1-scored-v2-1-0
             kwargs:
@@ -236,7 +237,7 @@ def schedule():
             returner: splunk_nova_return
             run_on_start: True
 
-    Note that ``args``, ``kwargs``, and ``splay`` are all optional. However, a
+    Note that ``args``, ``kwargs``,``min_splay`` and ``splay`` are all optional. However, a
     scheduled job must always have a ``function`` and a time in ``seconds`` of
     how often to run the job.
 
@@ -251,11 +252,16 @@ def schedule():
         Frequency with which the job should be run, in seconds
 
     splay
-        Randomized splay for the job, in seconds. A random number between 0 and
+        Randomized splay for the job, in seconds. A random number between <min_splay> and
         <splay> will be chosen and added to the ``seconds`` argument, to decide
         the true frequency. The splay will be chosen on first run, and will
         only change when the daemon is restarted. Optional.
 
+    min_splay
+        This parameters works in conjunction with <splay>. If a <min_splay> is provided, and random
+        between <min_splay> and <splay> is chosen. If <min_splay> is not provided, it 
+        defaults to zero. Optional.
+
     args
         List of arguments for the function. Optional.
 
@@ -294,6 +300,7 @@ def schedule():
             else:
                 seconds = int(jobdata['seconds'])
             splay = int(jobdata.get('splay', 0))
+            min_splay = int(jobdata.get('min_splay', 0))
         except ValueError:
             log.error('Scheduled job {0} has an invalid value for seconds or '
                       'splay.'.format(jobname))
@@ -317,7 +324,7 @@ def schedule():
                 if splay:
                     # Run `splay` seconds in the future, by telling the scheduler we last ran it
                     # `seconds - splay` seconds ago.
-                    jobdata['last_run'] = time.time() - (seconds - random.randint(0, splay))
+                    jobdata['last_run'] = time.time() - (seconds - random.randint(min_splay, splay))
                 else:
                     # Run now
                     run = True
@@ -326,7 +333,7 @@ def schedule():
                 if splay:
                     # Run `seconds + splay` seconds in the future by telling the scheduler we last
                     # ran it at now + `splay` seconds.
-                    jobdata['last_run'] = time.time() + random.randint(0, splay)
+                    jobdata['last_run'] = time.time() + random.randint(min_splay, splay)
                 elif 'buckets' in jobdata:
                     # Place the host in a bucket and fix the execution time.
                     jobdata['last_run'] = getlastrunbybuckets(jobdata['buckets'], seconds)
@@ -549,8 +556,15 @@ def load_config():
     ])
     __opts__['disable_modules'] = disable_modules
 
+    # Console logging is probably the same, but can be different
+    console_logging_opts = {
+        'log_level': __opts__.get('console_log_level', __opts__['log_level']),
+        'log_format': __opts__.get('console_log_format'),
+        'date_format': __opts__.get('console_log_date_format'),
+    }
+
     # Setup logging
-    salt.log.setup.setup_console_logger(__opts__['log_level'])
+    salt.log.setup.setup_console_logger(**console_logging_opts)
     salt.log.setup.setup_logfile_logger(__opts__['log_file'],
                                         __opts__['log_level'],
                                         max_bytes=__opts__.get('logfile_maxbytes', 100000000),

hubblestack/files/hubblestack_nova/misc.py

@@ -46,6 +46,7 @@
 import re
 import salt.utils
 from salt.ext import six
+from salt.exceptions import CommandExecutionError
 from collections import Counter
 
 log = logging.getLogger(__name__)
@@ -494,6 +495,9 @@ def check_directory_files_permission(path, permission):
     '''
     Check all files permission inside a directory
     '''
+    blacklisted_characters = '[^a-zA-Z0-9-_/]'
+    if "-exec" in path or re.findall(blacklisted_characters, path):
+      raise CommandExecutionError("Profile parameter '{0}' not a safe pattern".format(path))
     files_list = _execute_shell_command("find {0} -type f".format(path)).strip()
     files_list = files_list.split('\n') if files_list != "" else []
     bad_permission_files = []

hubblestack/files/hubblestack_nova/vulners_scanner.py

@@ -11,6 +11,7 @@
 The file should have the following format:
 
 vulners_scanner: <random data>
+vulners_api_key: REDACTED
 
 It does not matter what `<random data>` is, as long as the top key of the file is named `vulners_scanner`.
 This allows the module to run under a certain profile, as all of the other Nova modules do.
@@ -21,6 +22,7 @@
 
 import sys
 import requests
+import vulners
 
 
 log = logging.getLogger(__name__)
@@ -43,8 +45,8 @@ def audit(data_list, tags, labels, debug=False, **kwargs):
         if 'vulners_scanner' in data:
 
             local_packages = _get_local_packages()
-            vulners_data = _vulners_query(local_packages, os=os_name, version=os_version)
-            if vulners_data['result'] == 'ERROR':
+            vulners_data = _vulners_query(local_packages, os=os_name, version=os_version, api_key=data['vulners_api_key'])
+            if 'result' in vulners_data and vulners_data['result'] == 'ERROR':
                 log.error(vulners_data['data']['error'])
             vulners_data = _process_vulners(_vulners_query(local_packages, os=os_name, version=os_version))
 
@@ -69,7 +71,7 @@ def _get_local_packages():
     return ['{0}-{1}'.format(pkg, local_packages[pkg]) for pkg in local_packages]
 
 
-def _vulners_query(packages=None, os=None, version=None, url='https://vulners.com/api/v3/audit/audit/'):
+def _vulners_query(packages=None, os=None, version=None, api_key=None):
     '''
     Query the Vulners.com Linux Vulnerability Audit API for the provided packages.
 
@@ -101,24 +103,8 @@ def _vulners_query(packages=None, os=None, version=None, url='https://vulners.co
         error['data']['error'] = 'Missing the operating system version.'
         return error
 
-    headers = {
-        'Accept': 'application/json',
-        'Content-Type': 'application/json'
-    }
-
-    data = {
-        "os": os,
-        "package": packages,
-        "version": version
-    }
-
-    try:
-        response = requests.post(url=url, headers=headers, json=data)
-        return response.json()
-    except requests.Timeout:
-        error['data']['error'] = 'Request to {0} timed out'.format(url)
-        return error
-
+    vulners_api = vulners.Vulners(api_key=api_key)
+    return vulners_api.audit(str(os), str(version), packages)
 
 def _process_vulners(vulners):
     '''

pkg/amazonlinux2016.09/Dockerfile

@@ -125,8 +125,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
@@ -176,6 +176,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit
        --config-files /etc/osquery/osquery.conf \
        --after-install /hubble_build/conf/afterinstall.sh \
        --after-upgrade /hubble_build/conf/afterupgrade.sh \
+       --before-remove /hubble_build/conf/beforeremove.sh \
        etc/hubble etc/osquery etc/init.d opt usr \
 #edit to change iteration number, if necessary
     && cp hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.al1609.x86_64.rpm \

pkg/amazonlinux2016.09/pyinstaller-requirements.txt

@@ -16,3 +16,4 @@ azure
 azure-storage-common
 azure-storage-blob
 croniter
+vulners==1.3.0

pkg/centos6/Dockerfile

@@ -127,8 +127,8 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
@@ -178,6 +178,7 @@ CMD [ "scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINA
                                  --config-files /etc/osquery/osquery.conf \
                                  --after-install /hubble_build/conf/afterinstall.sh \
                                  --after-upgrade /hubble_build/conf/afterupgrade.sh \
+                                 --before-remove /hubble_build/conf/beforeremove.sh \
                                  etc/hubble etc/osquery etc/init.d opt usr' \
 #edit to change iteration number, if necessary
     && cp hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.el6.x86_64.rpm \

pkg/centos6/pyinstaller-requirements.txt

@@ -16,3 +16,4 @@ azure
 azure-storage-common
 azure-storage-blob
 croniter
+vulners==1.3.0

pkg/centos7/Dockerfile

@@ -124,8 +124,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src
@@ -175,6 +175,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit
        --config-files /etc/osquery/osquery.conf \
        --after-install /hubble_build/conf/afterinstall-systemd.sh \
        --after-upgrade /hubble_build/conf/afterupgrade-systemd.sh \
+       --before-remove /hubble_build/conf/beforeremove.sh \
        etc/hubble etc/osquery opt usr \
 #edit to change iteration number, if necessary
     && cp hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.x86_64.rpm /data/hubblestack-${HUBBLE_VERSION}-${HUBBLE_ITERATION}.el7.x86_64.rpm \

pkg/centos7/pyinstaller-requirements.txt

@@ -1,4 +1,4 @@
-pyinstaller==3.3.1 
+pyinstaller==3.3.1
 Crypto
 pyopenssl>=16.2.0
 argparse
@@ -16,3 +16,4 @@ azure
 azure-storage-common
 azure-storage-blob
 croniter
+vulners==1.3.0

pkg/coreos/Dockerfile

@@ -136,9 +136,9 @@ RUN pip -v install -r pyinstaller-requirements.txt
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/coreos/pyinstaller-requirements.txt

@@ -16,3 +16,4 @@ azure
 azure-storage-common
 azure-storage-blob
 croniter
+vulners==1.3.0

pkg/debian7/Dockerfile

@@ -163,9 +163,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"
@@ -217,6 +217,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit
        --deb-no-default-config-files \
        --after-install /hubble_build/conf/afterinstall.sh \
        --after-upgrade /hubble_build/conf/afterupgrade.sh \
+       --before-remove /hubble_build/conf/beforeremove.sh \
        etc/hubble etc/osquery etc/init.d opt usr \
     && cp hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}_amd64.deb /data/hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}deb7_amd64.deb \
     && openssl dgst -sha256 /data/hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}deb7_amd64.deb \

pkg/debian7/pyinstaller-requirements.txt

@@ -17,3 +17,4 @@ azure
 azure-storage-common
 azure-storage-blob
 croniter
+vulners==1.3.0

pkg/debian8/Dockerfile

@@ -145,9 +145,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"
@@ -200,6 +200,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit
        --deb-no-default-config-files \
        --after-install /hubble_build/conf/afterinstall.sh \
        --after-upgrade /hubble_build/conf/afterupgrade.sh \
+       --before-remove /hubble_build/conf/beforeremove.sh \
        etc/hubble etc/osquery etc/init.d opt usr \
     && cp hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}_amd64.deb /data/hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}deb8_amd64.deb \
     && openssl dgst -sha256 /data/hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}deb8_amd64.deb \

pkg/debian8/pyinstaller-requirements.txt

@@ -16,3 +16,4 @@ azure
 azure-storage-common
 azure-storage-blob
 croniter
+vulners==1.3.0

pkg/debian9/Dockerfile

@@ -140,9 +140,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.3
+ENV HUBBLE_CHECKOUT=v2.4.4
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.3
+ENV HUBBLE_VERSION=2.4.4
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"
@@ -196,6 +196,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit
        --deb-no-default-config-files \
        --after-install /hubble_build/conf/afterinstall.sh \
        --after-upgrade /hubble_build/conf/afterupgrade.sh \
+       --before-remove /hubble_build/conf/beforeremove.sh \
        etc/hubble etc/osquery etc/init.d opt usr \
     && cp hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}_amd64.deb /data/hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}deb9_amd64.deb \
     && openssl dgst -sha256 /data/hubblestack_${HUBBLE_VERSION}-${HUBBLE_ITERATION}deb9_amd64.deb \