From 514d71b6c9a9c94429575e7efad2f16490aad4a8 Mon Sep 17 00:00:00 2001
From: Colton Myers <colton.myers@gmail.com>
Date: Wed, 18 Oct 2017 15:10:42 -0600
Subject: [PATCH 1/6] Add azure_subscriptionId from upstream hubble

https://github.com/hubblestack/hubble/pull/227
---
 _returners/cloud_details.py        | 12 ++++++------
 _returners/splunk_nebula_return.py |  2 +-
 _returners/splunk_nova_return.py   |  2 +-
 _returners/splunk_pulsar_return.py |  2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/_returners/cloud_details.py b/_returners/cloud_details.py
index 7213669..422f26e 100644
--- a/_returners/cloud_details.py
+++ b/_returners/cloud_details.py
@@ -54,14 +54,14 @@ def _get_azure_details():
     # Gather azure information if present
     azure = {}
     azure['azure_vmId'] = None
-
+    azure['azure_subscriptionId'] = None
     azureHeader = {'Metadata': 'true'}
-
     try:
-        r = requests.get('http://169.254.169.254/metadata/instance/compute/vmId?api-version=2017-03-01&format=text',
-                         timeout=1, headers=azureHeader)
-        r.raise_for_status()
-        azure['azure_vmId'] = r.text
+        id = requests.get('http://169.254.169.254/metadata/instance/compute?api-version=2017-08-01',
+                          headers=azureHeader, timeout=1).json()
+        azure['azure_vmId'] = id['vmId']
+        azure['azure_subscriptionId'] = id['subscriptionId']
+
     except (requests.exceptions.RequestException, ValueError):
         # Not on an Azure box
         azure = None
diff --git a/_returners/splunk_nebula_return.py b/_returners/splunk_nebula_return.py
index 6c6bd6d..c97c937 100644
--- a/_returners/splunk_nebula_return.py
+++ b/_returners/splunk_nebula_return.py
@@ -80,7 +80,7 @@ def returner(ret):
 
             # Set up the fields to be extracted at index time. The field values must be strings.
             # Note that these fields will also still be available in the event data
-            index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId']
+            index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId', 'azure_subscriptionId']
             try:
                 index_extracted_fields.extend(opts['index_extracted_fields'])
             except TypeError:
diff --git a/_returners/splunk_nova_return.py b/_returners/splunk_nova_return.py
index f457429..6338b61 100644
--- a/_returners/splunk_nova_return.py
+++ b/_returners/splunk_nova_return.py
@@ -79,7 +79,7 @@ def returner(ret):
 
             # Set up the fields to be extracted at index time. The field values must be strings.
             # Note that these fields will also still be available in the event data
-            index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId']
+            index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId', 'azure_subscriptionId']
             try:
                 index_extracted_fields.extend(opts['index_extracted_fields'])
             except TypeError:
diff --git a/_returners/splunk_pulsar_return.py b/_returners/splunk_pulsar_return.py
index ad0b0b5..d7d54a4 100644
--- a/_returners/splunk_pulsar_return.py
+++ b/_returners/splunk_pulsar_return.py
@@ -84,7 +84,7 @@ def returner(ret):
 
             # Set up the fields to be extracted at index time. The field values must be strings.
             # Note that these fields will also still be available in the event data
-            index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId']
+            index_extracted_fields = ['aws_instance_id', 'aws_account_id', 'azure_vmId', 'azure_subscriptionId']
             try:
                 index_extracted_fields.extend(opts['index_extracted_fields'])
             except TypeError:

From 8274ca7dfd2a570b47d6861d377c006174ea64cc Mon Sep 17 00:00:00 2001
From: Colton Myers <colton.myers@gmail.com>
Date: Mon, 23 Oct 2017 10:38:40 -0600
Subject: [PATCH 2/6] Add fix from upstream hubble

https://github.com/hubblestack/hubble/pull/229
---
 hubblestack_nova/win_secedit.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/hubblestack_nova/win_secedit.py b/hubblestack_nova/win_secedit.py
index 70bd1f5..c9b120a 100644
--- a/hubblestack_nova/win_secedit.py
+++ b/hubblestack_nova/win_secedit.py
@@ -283,7 +283,9 @@ def _translate_value_type(current, value, evaluator, __sidaccounts__=False):
             return False
     elif 'equal' in value:
         if ',' not in evaluator and type(evaluator) != list:
-            evaluator = _evaluator_translator(evaluator)
+            tmp_evaluator = _evaluator_translator(evaluator)
+            if tmp_evaluator != 'undefined':
+                evaluator = tmp_evaluator
         if type(current) == list:
             ret_final = []
             for item in current:
@@ -300,6 +302,21 @@ def _translate_value_type(current, value, evaluator, __sidaccounts__=False):
             return True
         else:
             return False
+    elif 'contains' in value:
+        if type(evaluator) != list:
+            evaluator = evaluator.split(',')
+            if type(current) != list:
+                current = current.lower().split(',')
+            ret_final = []
+            for item in evaluator:
+                if item in current:
+                    ret_final.append(True)
+                else:
+                    ret_final.append(False)
+            if False in ret_final:
+                return False
+            else:
+                return True
     elif 'account' in value:
         evaluator = _account_audit(evaluator, __sidaccounts__)
         evaluator_list = evaluator.split(',')
@@ -327,7 +344,7 @@ def _translate_value_type(current, value, evaluator, __sidaccounts__=False):
     elif 'configured' in value:
         if current == '':
             return False
-        elif current == value:
+        elif current.lower().find(evaluator) != -1:
             return True
         else:
             return False

From b1eb94579a8f585c2a8d83a2468f9704915d78e9 Mon Sep 17 00:00:00 2001
From: Chandler Newby <chandler.newby@gmail.com>
Date: Fri, 3 Nov 2017 13:43:12 -0600
Subject: [PATCH 3/6] Remove empty fields from splunk returners

---
 _returners/splunk_nebula_return.py | 6 ++++++
 _returners/splunk_nova_return.py   | 6 ++++++
 _returners/splunk_pulsar_return.py | 6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/_returners/splunk_nebula_return.py b/_returners/splunk_nebula_return.py
index c97c937..38022f6 100644
--- a/_returners/splunk_nebula_return.py
+++ b/_returners/splunk_nebula_return.py
@@ -141,6 +141,12 @@ def returner(ret):
                                 payload.update({'sourcetype': "%s_%s" % (opts['sourcetype'], query_name)})
                             else:
                                 payload.update({'sourcetype': opts['sourcetype']})
+
+                            # Remove any empty fields from the event payload
+                            remove_keys = [k for k in event if event[k] == ""]
+                            for k in remove_keys:
+                                del event[k]
+
                             payload.update({'event': event})
 
                             # Potentially add metadata fields:
diff --git a/_returners/splunk_nova_return.py b/_returners/splunk_nova_return.py
index 6338b61..275b60f 100644
--- a/_returners/splunk_nova_return.py
+++ b/_returners/splunk_nova_return.py
@@ -193,6 +193,12 @@ def returner(ret):
                 payload.update({'host': fqdn})
                 payload.update({'sourcetype': opts['sourcetype']})
                 payload.update({'index': opts['index']})
+
+                # Remove any empty fields from the event payload
+                remove_keys = [k for k in event if event[k] == ""]
+                for k in remove_keys:
+                    del event[k]
+
                 payload.update({'event': event})
 
                 # Potentially add metadata fields:
diff --git a/_returners/splunk_pulsar_return.py b/_returners/splunk_pulsar_return.py
index d7d54a4..de7736c 100644
--- a/_returners/splunk_pulsar_return.py
+++ b/_returners/splunk_pulsar_return.py
@@ -232,6 +232,12 @@ def returner(ret):
                 payload.update({'host': fqdn})
                 payload.update({'index': opts['index']})
                 payload.update({'sourcetype': opts['sourcetype']})
+
+                # Remove any empty fields from the event payload
+                remove_keys = [k for k in event if event[k] == ""]
+                for k in remove_keys:
+                    del event[k]
+
                 payload.update({'event': event})
 
                 # Potentially add metadata fields:

From 5a90f4c28c5b6b0926cd6ecc59542b350675ccd8 Mon Sep 17 00:00:00 2001
From: Colton Myers <colton.myers@gmail.com>
Date: Tue, 7 Nov 2017 10:43:00 -0700
Subject: [PATCH 4/6] Add changes from upstream hubble

https://github.com/hubblestack/hubble/pull/244
---
 _modules/nebula_osquery.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/_modules/nebula_osquery.py b/_modules/nebula_osquery.py
index a0e3ad5..d122c20 100644
--- a/_modules/nebula_osquery.py
+++ b/_modules/nebula_osquery.py
@@ -199,10 +199,11 @@ def queries(query_group,
 
     for r in ret:
         for query_name, query_ret in r.iteritems():
-            for result in query_ret['data']:
-                for key, value in result.iteritems():
-                    if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'):
-                        result[key] = json.loads(value[len('__JSONIFY__'):])
+            if 'data' in query_ret:
+                for result in query_ret['data']:
+                    for key, value in result.iteritems():
+                        if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'):
+                            result[key] = json.loads(value[len('__JSONIFY__'):])
 
     return ret
 

From c07f74eae34c2a27eefc578e859e86d5c1739590 Mon Sep 17 00:00:00 2001
From: Colton Myers <colton.myers@gmail.com>
Date: Thu, 16 Nov 2017 14:18:59 -0700
Subject: [PATCH 5/6] Pull in salt.utils.context for backwards compatibility
 with 2015.8

---
 _modules/nova_loader.py | 249 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 243 insertions(+), 6 deletions(-)

diff --git a/_modules/nova_loader.py b/_modules/nova_loader.py
index e28603b..e498907 100644
--- a/_modules/nova_loader.py
+++ b/_modules/nova_loader.py
@@ -26,7 +26,6 @@
 from salt.template import check_render_pipe_str
 from salt.utils.decorators import Depends
 from salt.utils import is_proxy
-import salt.utils.context
 import salt.utils.lazy
 import salt.utils.event
 import salt.utils.odict
@@ -1147,7 +1146,7 @@ def __init__(self,
         self.pack = {} if pack is None else pack
         if opts is None:
             opts = {}
-        self.context_dict = salt.utils.context.ContextDict()
+        self.context_dict = ContextDict()
         self.opts = self.__prep_mod_opts(opts)
 
         self.module_dirs = module_dirs
@@ -1161,7 +1160,7 @@ def __init__(self,
         for k, v in six.iteritems(self.pack):
             if v is None:  # if the value of a pack is None, lets make an empty dict
                 self.context_dict.setdefault(k, {})
-                self.pack[k] = salt.utils.context.NamespacedDictWrapper(self.context_dict, k)
+                self.pack[k] = NamespacedDictWrapper(self.context_dict, k)
 
         self.whitelist = whitelist
         self.virtual_enable = virtual_enable
@@ -1353,11 +1352,11 @@ def __prep_mod_opts(self, opts):
         '''
         if '__grains__' not in self.pack:
             self.context_dict['grains'] = opts.get('grains', {})
-            self.pack['__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'grains', override_name='grains')
+            self.pack['__grains__'] = NamespacedDictWrapper(self.context_dict, 'grains', override_name='grains')
 
         if '__pillar__' not in self.pack:
             self.context_dict['pillar'] = opts.get('pillar', {})
-            self.pack['__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'pillar', override_name='pillar')
+            self.pack['__pillar__'] = NamespacedDictWrapper(self.context_dict, 'pillar', override_name='pillar')
 
         mod_opts = {}
         for key, val in list(opts.items()):
@@ -1809,7 +1808,7 @@ def global_injector_decorator(inject_globals):
     def inner_decorator(f):
         @functools.wraps(f)
         def wrapper(*args, **kwargs):
-            with salt.utils.context.func_globals_inject(f, **inject_globals):
+            with func_globals_inject(f, **inject_globals):
                 return f(*args, **kwargs)
         return wrapper
     return inner_decorator
@@ -2045,3 +2044,241 @@ def _load_module(self, name):
 
         self.loaded_modules[name] = mod_dict
         return True
+
+
+#############################################################################
+# salt.utils.context from newer version of salt for compatibility with 2015.8
+#############################################################################
+
+# Import python libs
+import copy
+import threading
+import collections
+from contextlib import contextmanager
+
+from salt.ext import six
+
+
+@contextmanager
+def func_globals_inject(func, **overrides):
+    '''
+    Override specific variables within a function's global context.
+    '''
+    # recognize methods
+    if hasattr(func, 'im_func'):
+        func = func.__func__
+
+    # Get a reference to the function globals dictionary
+    func_globals = func.__globals__
+    # Save the current function globals dictionary state values for the
+    # overridden objects
+    injected_func_globals = []
+    overridden_func_globals = {}
+    for override in overrides:
+        if override in func_globals:
+            overridden_func_globals[override] = func_globals[override]
+        else:
+            injected_func_globals.append(override)
+
+    # Override the function globals with what's passed in the above overrides
+    func_globals.update(overrides)
+
+    # The context is now ready to be used
+    yield
+
+    # We're now done with the context
+
+    # Restore the overwritten function globals
+    func_globals.update(overridden_func_globals)
+
+    # Remove any entry injected in the function globals
+    for injected in injected_func_globals:
+        del func_globals[injected]
+
+
+class ContextDict(collections.MutableMapping):
+    '''
+    A context manager that saves some per-thread state globally.
+    Intended for use with Tornado's StackContext.
+
+    Provide arbitrary data as kwargs upon creation,
+    then allow any children to override the values of the parent.
+    '''
+
+    def __init__(self, threadsafe=False, **data):
+        # state should be thread local, so this object can be threadsafe
+        self._state = threading.local()
+        # variable for the overridden data
+        self._state.data = None
+        self.global_data = {}
+        # Threadsafety indicates whether or not we should protect data stored
+        # in child context dicts from being leaked
+        self._threadsafe = threadsafe
+
+    @property
+    def active(self):
+        '''Determine if this ContextDict is currently overridden
+        Since the ContextDict can be overridden in each thread, we check whether
+        the _state.data is set or not.
+        '''
+        try:
+            return self._state.data is not None
+        except AttributeError:
+            return False
+
+    # TODO: rename?
+    def clone(self, **kwargs):
+        '''
+        Clone this context, and return the ChildContextDict
+        '''
+        child = ChildContextDict(parent=self, threadsafe=self._threadsafe, overrides=kwargs)
+        return child
+
+    def __setitem__(self, key, val):
+        if self.active:
+            self._state.data[key] = val
+        else:
+            self.global_data[key] = val
+
+    def __delitem__(self, key):
+        if self.active:
+            del self._state.data[key]
+        else:
+            del self.global_data[key]
+
+    def __getitem__(self, key):
+        if self.active:
+            return self._state.data[key]
+        else:
+            return self.global_data[key]
+
+    def __len__(self):
+        if self.active:
+            return len(self._state.data)
+        else:
+            return len(self.global_data)
+
+    def __iter__(self):
+        if self.active:
+            return iter(self._state.data)
+        else:
+            return iter(self.global_data)
+
+    def __copy__(self):
+        new_obj = type(self)(threadsafe=self._threadsafe)
+        if self.active:
+            new_obj.global_data = copy.copy(self._state.data)
+        else:
+            new_obj.global_data = copy.copy(self.global_data)
+        return new_obj
+
+    def __deepcopy__(self, memo):
+        new_obj = type(self)(threadsafe=self._threadsafe)
+        if self.active:
+            new_obj.global_data = copy.deepcopy(self._state.data, memo)
+        else:
+            new_obj.global_data = copy.deepcopy(self.global_data, memo)
+        return new_obj
+
+
+class ChildContextDict(collections.MutableMapping):
+    '''An overrideable child of ContextDict
+
+    '''
+    def __init__(self, parent, overrides=None, threadsafe=False):
+        self.parent = parent
+        self._data = {} if overrides is None else overrides
+        self._old_data = None
+
+        # merge self.global_data into self._data
+        if threadsafe:
+            for k, v in six.iteritems(self.parent.global_data):
+                if k not in self._data:
+                    # A deepcopy is necessary to avoid using the same
+                    # objects in globals as we do in thread local storage.
+                    # Otherwise, changing one would automatically affect
+                    # the other.
+                    self._data[k] = copy.deepcopy(v)
+        else:
+            for k, v in six.iteritems(self.parent.global_data):
+                if k not in self._data:
+                    self._data[k] = v
+
+    def __setitem__(self, key, val):
+        self._data[key] = val
+
+    def __delitem__(self, key):
+        del self._data[key]
+
+    def __getitem__(self, key):
+        return self._data[key]
+
+    def __len__(self):
+        return len(self._data)
+
+    def __iter__(self):
+        return iter(self._data)
+
+    def __enter__(self):
+        if hasattr(self.parent._state, 'data'):
+            # Save old data to support nested calls
+            self._old_data = self.parent._state.data
+        self.parent._state.data = self._data
+
+    def __exit__(self, *exc):
+        self.parent._state.data = self._old_data
+
+
+class NamespacedDictWrapper(collections.MutableMapping, dict):
+    '''
+    Create a dict which wraps another dict with a specific prefix of key(s)
+
+    MUST inherit from dict to serialize through msgpack correctly
+    '''
+
+    def __init__(self, d, pre_keys, override_name=None):  # pylint: disable=W0231
+        self.__dict = d
+        if isinstance(pre_keys, six.string_types):
+            self.pre_keys = (pre_keys,)
+        else:
+            self.pre_keys = pre_keys
+        if override_name:
+            self.__class__.__module__ = 'salt'
+            # __name__ can't be assigned a unicode
+            self.__class__.__name__ = str(override_name)  # future lint: disable=non-unicode-string
+        super(NamespacedDictWrapper, self).__init__(self._dict())
+
+    def _dict(self):
+        r = self.__dict
+        for k in self.pre_keys:
+            r = r[k]
+        return r
+
+    def __repr__(self):
+        return repr(self._dict())
+
+    def __setitem__(self, key, val):
+        self._dict()[key] = val
+
+    def __delitem__(self, key):
+        del self._dict()[key]
+
+    def __getitem__(self, key):
+        return self._dict()[key]
+
+    def __len__(self):
+        return len(self._dict())
+
+    def __iter__(self):
+        return iter(self._dict())
+
+    def __copy__(self):
+        return type(self)(copy.copy(self.__dict),
+                          copy.copy(self.pre_keys))
+
+    def __deepcopy__(self, memo):
+        return type(self)(copy.deepcopy(self.__dict, memo),
+                          copy.deepcopy(self.pre_keys, memo))
+
+    def __str__(self):
+        return self._dict().__str__()

From c161e82c5c5205f5116c6dbf7503d948f345ffd6 Mon Sep 17 00:00:00 2001
From: Colton Myers <colton.myers@gmail.com>
Date: Mon, 20 Nov 2017 14:31:13 -0700
Subject: [PATCH 6/6] Rev to v2017.11.0

---
 README.md                           | 2 +-
 _beacons/pulsar.py                  | 2 +-
 _modules/hubble.py                  | 2 +-
 _modules/nebula_osquery.py          | 2 +-
 _modules/win_pulsar.py              | 2 +-
 _returners/slack_pulsar_returner.py | 2 +-
 _returners/splunk_nebula_return.py  | 2 +-
 _returners/splunk_nova_return.py    | 2 +-
 _returners/splunk_pulsar_return.py  | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 42d3ee9..6eb6558 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ gitfs_remotes:
   - https://github.com/hubblestack/hubblestack_data.git:
     - root: ''
   - https://github.com/hubblestack/hubble-salt.git:
-    - base: v2017.9.2
+    - base: v2017.11.0
     - root: ''
 ```
 
diff --git a/_beacons/pulsar.py b/_beacons/pulsar.py
index 614eafc..6a0e045 100644
--- a/_beacons/pulsar.py
+++ b/_beacons/pulsar.py
@@ -39,7 +39,7 @@
     DEFAULT_MASK = None
 
 __virtualname__ = 'pulsar'
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 CONFIG = None
 CONFIG_STALENESS = 0
 
diff --git a/_modules/hubble.py b/_modules/hubble.py
index e9fe578..d7e2688 100644
--- a/_modules/hubble.py
+++ b/_modules/hubble.py
@@ -35,7 +35,7 @@
 from nova_loader import NovaLazyLoader
 
 __nova__ = {}
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 
 
 def audit(configs=None,
diff --git a/_modules/nebula_osquery.py b/_modules/nebula_osquery.py
index d122c20..d8a8972 100644
--- a/_modules/nebula_osquery.py
+++ b/_modules/nebula_osquery.py
@@ -41,7 +41,7 @@
 
 log = logging.getLogger(__name__)
 
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 __virtualname__ = 'nebula'
 
 
diff --git a/_modules/win_pulsar.py b/_modules/win_pulsar.py
index d2f30f0..ed8c115 100644
--- a/_modules/win_pulsar.py
+++ b/_modules/win_pulsar.py
@@ -28,7 +28,7 @@
 CONFIG = None
 CONFIG_STALENESS = 0
 
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 
 
 def __virtual__():
diff --git a/_returners/slack_pulsar_returner.py b/_returners/slack_pulsar_returner.py
index a3fdba8..c087d47 100644
--- a/_returners/slack_pulsar_returner.py
+++ b/_returners/slack_pulsar_returner.py
@@ -69,7 +69,7 @@
 # Import Salt Libs
 import salt.returners
 
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 
 log = logging.getLogger(__name__)
 
diff --git a/_returners/splunk_nebula_return.py b/_returners/splunk_nebula_return.py
index 38022f6..63ad477 100644
--- a/_returners/splunk_nebula_return.py
+++ b/_returners/splunk_nebula_return.py
@@ -50,7 +50,7 @@
 
 import logging
 
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
diff --git a/_returners/splunk_nova_return.py b/_returners/splunk_nova_return.py
index 275b60f..7468537 100644
--- a/_returners/splunk_nova_return.py
+++ b/_returners/splunk_nova_return.py
@@ -49,7 +49,7 @@
 
 import logging
 
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False
diff --git a/_returners/splunk_pulsar_return.py b/_returners/splunk_pulsar_return.py
index de7736c..8829c5d 100644
--- a/_returners/splunk_pulsar_return.py
+++ b/_returners/splunk_pulsar_return.py
@@ -51,7 +51,7 @@
 
 import logging
 
-__version__ = 'v2017.9.2'
+__version__ = 'v2017.11.0'
 
 _max_content_bytes = 100000
 http_event_collector_SSL_verify = False