From d007730f50061093298a2b64ecc2613d7ffcc5fe Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Thu, 26 Jul 2018 15:05:30 +0530 Subject: [PATCH 01/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 573b7545a..a1efdc280 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -32,6 +32,7 @@ import json import logging import os +import re import time import yaml import collections @@ -318,6 +319,54 @@ def get_top_data(topfile): return ret +def mask_passwords(unmasked_object): + + masked_object = copy.deepcopy(unmasked_object) + + try: + mask = yaml.load(open('mask.yaml')) + mask_by = mask.get('mask_by','******') + for r in masked_object: + for query_name, query_ret in r.iteritems(): + if 'data' in query_ret: + for result in query_ret['data']: + for key, value in result.iteritems(): + if isinstance(value,basestring): + # handle like string + for blacklisted_string in mask.get("blacklisted_strings",[]): + if blacklisted_string['query_name'] in ('*', query_name) and \ + key == blacklisted_string['column']: + for pattern in blacklisted_string['blacklisted_patterns']: + value = re.sub(pattern + "()",r"\1" + mask_by + r"\3", value) + #print("string masking : {0} : {1}".format(result[key],value)) + result[key] = value + else: + # handle like [json] + for blacklisted_object in mask.get('blacklisted_objects',[]): + if blacklisted_object['query_name'] in ('*', query_name) and \ + key == blacklisted_object['column']: + _recursively_mask_objects(value, blacklisted_object, mask_by) + + return masked_object + + except Exception as e: + print("An error occured while masking the passwords: {}".format(e)) + return unmasked_object + + +def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by): + + if isinstance(object_to_mask, list): + for child in object_to_mask: + _recursively_mask_objects(child, blacklisted_object, mask_by) + elif blacklisted_object['attribute_to_check'] in object_to_mask and \ + object_to_mask[blacklisted_object['attribute_to_check']] in blacklisted_object['balcklisted_patterns']: + for key in blacklisted_object['attributes_to_mask']: + if key in object_to_mask: + object_to_mask[key] = mask_by + print(" masking : {0}".format(object_to_mask[key])) + + def _dict_update(dest, upd, recursive_update=True, merge_lists=False): ''' Recursive version of the default dict.update From 77c6edc66431184b5dd73b1b89ef27d820ec2f45 Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Thu, 26 Jul 2018 15:08:03 +0530 Subject: [PATCH 02/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index a1efdc280..fc74827a0 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -226,7 +226,7 @@ def queries(query_group, if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'): result[key] = json.loads(value[len('__JSONIFY__'):]) - return ret + return mask_passwords(ret) def fields(*args): From 2df78dab45d4cf5387d195d1ff3b1f93d0246e5c Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Thu, 26 Jul 2018 15:53:45 +0530 Subject: [PATCH 03/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index fc74827a0..c2a9c31a1 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -324,7 +324,7 @@ def mask_passwords(unmasked_object): masked_object = copy.deepcopy(unmasked_object) try: - mask = yaml.load(open('mask.yaml')) + mask = yaml.load(open('salt://mask.yaml')) mask_by = mask.get('mask_by','******') for r in masked_object: for query_name, query_ret in r.iteritems(): From 4e49a704d225f7591cb7c400dad5cb0b54e26f6e Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Thu, 26 Jul 2018 16:55:22 +0530 Subject: [PATCH 04/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index c2a9c31a1..2711e1998 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -324,7 +324,8 @@ def mask_passwords(unmasked_object): masked_object = copy.deepcopy(unmasked_object) try: - mask = yaml.load(open('salt://mask.yaml')) + mask_file = __salt__['cp.cache_file']('salt://mask.yaml') + mask = yaml.safe_load(open(mask_file)) mask_by = mask.get('mask_by','******') for r in masked_object: for query_name, query_ret in r.iteritems(): From c036c8554fe987d4c9526461c3ce38d57760445c Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Thu, 26 Jul 2018 17:07:34 +0530 Subject: [PATCH 05/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 2711e1998..ec971c20c 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -361,7 +361,7 @@ def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by): for child in object_to_mask: _recursively_mask_objects(child, blacklisted_object, mask_by) elif blacklisted_object['attribute_to_check'] in object_to_mask and \ - object_to_mask[blacklisted_object['attribute_to_check']] in blacklisted_object['balcklisted_patterns']: + object_to_mask[blacklisted_object['attribute_to_check']] in blacklisted_object['blacklisted_patterns']: for key in blacklisted_object['attributes_to_mask']: if key in object_to_mask: object_to_mask[key] = mask_by From bbebd3f05990a93360bfc3f5d395348fdb1afd7c Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Thu, 26 Jul 2018 17:10:03 +0530 Subject: [PATCH 06/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index ec971c20c..eb95b523a 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -339,7 +339,6 @@ def mask_passwords(unmasked_object): key == blacklisted_string['column']: for pattern in blacklisted_string['blacklisted_patterns']: value = re.sub(pattern + "()",r"\1" + mask_by + r"\3", value) - #print("string masking : {0} : {1}".format(result[key],value)) result[key] = value else: # handle like [json] @@ -365,7 +364,6 @@ def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by): for key in blacklisted_object['attributes_to_mask']: if key in object_to_mask: object_to_mask[key] = mask_by - print(" masking : {0}".format(object_to_mask[key])) def _dict_update(dest, upd, recursive_update=True, merge_lists=False): From 653c96daa6bafe9ae94207689323cd34fa1d3321 Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Fri, 27 Jul 2018 12:08:49 +0530 Subject: [PATCH 07/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index eb95b523a..f20d63eb6 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -55,7 +55,8 @@ def __virtual__(): def queries(query_group, query_file=None, verbose=False, - report_version_with_day=True): + report_version_with_day=True, + mask_passwords=False): ''' Run the set of queries represented by ``query_group`` from the configuration in the file query_file @@ -226,7 +227,7 @@ def queries(query_group, if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'): result[key] = json.loads(value[len('__JSONIFY__'):]) - return mask_passwords(ret) + return mask_passwords(ret) if mask_passwords else ret def fields(*args): @@ -274,7 +275,8 @@ def hubble_versions(): def top(query_group, topfile='salt://hubblestack_nebula_v2/top.nebula', verbose=False, - report_version_with_day=True): + report_version_with_day=True, + mask_passwords=False): if salt.utils.platform.is_windows(): topfile = 'salt://hubblestack_nebula_v2/win_top.nebula' @@ -287,7 +289,8 @@ def top(query_group, return queries(query_group, query_file=configs, verbose=False, - report_version_with_day=True) + report_version_with_day=True, + mask_passwords=mask_passwords) def get_top_data(topfile): @@ -319,39 +322,38 @@ def get_top_data(topfile): return ret -def mask_passwords(unmasked_object): - - masked_object = copy.deepcopy(unmasked_object) +def mask_passwords(object_to_be_masked): try: - mask_file = __salt__['cp.cache_file']('salt://mask.yaml') + mask_file = __salt__['cp.cache_file']('salt://hubblestack_nebula_v2/mask.yaml') + if not mask_file: + return mask = yaml.safe_load(open(mask_file)) mask_by = mask.get('mask_by','******') - for r in masked_object: + for r in object_to_be_masked: for query_name, query_ret in r.iteritems(): if 'data' in query_ret: for result in query_ret['data']: for key, value in result.iteritems(): - if isinstance(value,basestring): + if isinstance(value, basestring): # handle like string - for blacklisted_string in mask.get("blacklisted_strings",[]): + for blacklisted_string in mask.get("blacklisted_strings", []): if blacklisted_string['query_name'] in ('*', query_name) and \ key == blacklisted_string['column']: for pattern in blacklisted_string['blacklisted_patterns']: - value = re.sub(pattern + "()",r"\1" + mask_by + r"\3", value) + value = re.sub(pattern + "()", r"\1" + mask_by + r"\3", value) result[key] = value else: # handle like [json] - for blacklisted_object in mask.get('blacklisted_objects',[]): + for blacklisted_object in mask.get('blacklisted_objects', []): if blacklisted_object['query_name'] in ('*', query_name) and \ key == blacklisted_object['column']: _recursively_mask_objects(value, blacklisted_object, mask_by) - - return masked_object - + + # successfully masked the object. No need to return anything + except Exception as e: - print("An error occured while masking the passwords: {}".format(e)) - return unmasked_object + log.exception("An error occured while masking the passwords: {}".format(e)) def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by): From 20f23e1f28e2f83cfdd0db167a04e9a2317e15a6 Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Fri, 27 Jul 2018 12:21:18 +0530 Subject: [PATCH 08/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index f20d63eb6..78815502f 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -227,7 +227,9 @@ def queries(query_group, if value and isinstance(value, basestring) and value.startswith('__JSONIFY__'): result[key] = json.loads(value[len('__JSONIFY__'):]) - return mask_passwords(ret) if mask_passwords else ret + if mask_passwords: + mask_passwords_inplace(ret) + return ret def fields(*args): @@ -322,7 +324,7 @@ def get_top_data(topfile): return ret -def mask_passwords(object_to_be_masked): +def mask_passwords_inplace(object_to_be_masked): try: mask_file = __salt__['cp.cache_file']('salt://hubblestack_nebula_v2/mask.yaml') From 04fa3964cee67a3266945d4d0ba7b3b3d1dd77aa Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Fri, 27 Jul 2018 12:55:47 +0530 Subject: [PATCH 09/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 78815502f..d4800239d 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -71,6 +71,10 @@ def queries(query_group, Defaults to False. If set to True, more information (such as the query which was run) will be included in the result. + mask_passwords + Defaults to False. If set to True, passwords mentioned in the + return object are masked. + CLI Examples: .. code_block:: bash @@ -325,7 +329,11 @@ def get_top_data(topfile): def mask_passwords_inplace(object_to_be_masked): - + ''' + It masks the passwords present in 'object_to_be_masked'. Uses "mask.yaml" as + a reference to find out the list of blacklisted strings or objects. + Note that this method alters "object_to_be_masked". + ''' try: mask_file = __salt__['cp.cache_file']('salt://hubblestack_nebula_v2/mask.yaml') if not mask_file: @@ -359,7 +367,24 @@ def mask_passwords_inplace(object_to_be_masked): def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by): - + ''' + This function is used by "mask_passwords_inplace" to mask passwords contained in + json objects or json arrays. If the "object_to_mask" is a json array, then this + function is called recursively on the individual members of the array. + + object_to_mask + Json object/array whose elements are to masked recursively + + blacklisted_object + This parameters contains info about which queries are to be masked, which + attributes are to be masked, based upon the value of which attribute. + See hubblestack_nebula_v2/mask.yaml for exact format. + + mask_by + If a password string is detected, it is replaced by the value of "mask_by" + parameter. + + ''' if isinstance(object_to_mask, list): for child in object_to_mask: _recursively_mask_objects(child, blacklisted_object, mask_by) From c59935ebec23690481b86b120a2043e2cfe044ad Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Mon, 30 Jul 2018 18:06:07 +0530 Subject: [PATCH 10/23] Avoiding loops. Adding top.mask. --- hubblestack/extmods/modules/nebula_osquery.py | 105 +++++++++++++----- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index d4800239d..3c965ae9a 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -335,32 +335,87 @@ def mask_passwords_inplace(object_to_be_masked): Note that this method alters "object_to_be_masked". ''' try: - mask_file = __salt__['cp.cache_file']('salt://hubblestack_nebula_v2/mask.yaml') - if not mask_file: - return - mask = yaml.safe_load(open(mask_file)) + + mask = {} + topfile = 'salt://hubblestack_nebula_v2/top.mask' + mask_files = get_top_data(topfile) + mask_files = ['salt://hubblestack_nebula_v2/' + mask_file.replace('.', '/') + '.yaml' + for mask_file in mask_files] + if mask_files is None: + mask_files = 'salt://hubblestack_nebula_v2/mask.yaml' + if not isinstance(mask_files, list): + mask_files = [mask_files] + for fh in mask_files: + if 'salt://' in fh: + orig_fh = fh + fh = __salt__['cp.cache_file'](fh) + if fh is None: + log.error('Could not find file {0}.'.format(orig_fh)) + return None + if os.path.isfile(fh): + with open(fh, 'r') as f: + f_data = yaml.safe_load(f) + if not isinstance(f_data, dict): + raise CommandExecutionError('File data is not formed as a dict {0}' + .format(f_data)) + mask = _dict_update(mask, f_data, recursive_update=True, merge_lists=True) + + mask_by = mask.get('mask_by','******') - for r in object_to_be_masked: - for query_name, query_ret in r.iteritems(): - if 'data' in query_ret: - for result in query_ret['data']: - for key, value in result.iteritems(): - if isinstance(value, basestring): - # handle like string - for blacklisted_string in mask.get("blacklisted_strings", []): - if blacklisted_string['query_name'] in ('*', query_name) and \ - key == blacklisted_string['column']: - for pattern in blacklisted_string['blacklisted_patterns']: - value = re.sub(pattern + "()", r"\1" + mask_by + r"\3", value) - result[key] = value - else: - # handle like [json] - for blacklisted_object in mask.get('blacklisted_objects', []): - if blacklisted_object['query_name'] in ('*', query_name) and \ - key == blacklisted_object['column']: - _recursively_mask_objects(value, blacklisted_object, mask_by) - - # successfully masked the object. No need to return anything + + + for blacklisted_string in mask.get("blacklisted_strings", []): + query_name = blacklisted_string['query_name'] + column = blacklisted_string['column'] + if query_name != '*': + for r in object_to_be_masked: + for query_result in r.get(query_name,{'data':[]})['data']: + if column not in query_result or not isinstance(query_result[column], basestring): + # if the column in not present in one data-object, it will + # not be present in others as well. Break in that case. + # This will happen only if mask.yaml is malformed + print("string", query_name, query_result, column) + break + value = query_result[column] + for pattern in blacklisted_string['blacklisted_patterns']: + value = re.sub(pattern + "()", r"\1" + mask_by + r"\3", value) + query_result[column] = value + else: + for r in object_to_be_masked: + for query_name, query_ret in r.iteritems(): + for query_result in query_ret['data']: + if column not in query_result or not isinstance(query_result[column], basestring): + print("string", query_name, query_result, column) + break + value = query_result[column] + for pattern in blacklisted_string['blacklisted_patterns']: + value = re.sub(pattern + "()", r"\1" + mask_by + r"\3", value) + query_result[column] = value + + + for blacklisted_object in mask.get("blacklisted_objects", []): + query_name = blacklisted_object['query_name'] + column = blacklisted_object['column'] + if query_name != '*': + for r in object_to_be_masked: + for query_result in r.get(query_name,{'data':[]})['data']: + if column not in query_result or \ + ( isinstance(query_result[column], basestring) and query_result[column].strip() != '' ): + print("object", query_name, query_result, column) + break + _recursively_mask_objects(query_result[column], blacklisted_object, mask_by) + else: + for r in object_to_be_masked: + for query_name, query_ret in r.iteritems(): + for query_result in query_ret['data']: + if column not in query_result or \ + ( isinstance(query_result[column], basestring) and query_result[column].strip() != '' ): + print("object", query_name, query_result, column) + break + _recursively_mask_objects(query_result[column], blacklisted_object, mask_by) + + + # successfully masked the object. No need to return anything except Exception as e: log.exception("An error occured while masking the passwords: {}".format(e)) From 63c243c53d71cae4bfd2bd4e7a120b31b671dbaa Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Mon, 30 Jul 2018 18:27:33 +0530 Subject: [PATCH 11/23] Removing some print statements --- hubblestack/extmods/modules/nebula_osquery.py | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 3c965ae9a..3299b4c1a 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -327,7 +327,6 @@ def get_top_data(topfile): return ret - def mask_passwords_inplace(object_to_be_masked): ''' It masks the passwords present in 'object_to_be_masked'. Uses "mask.yaml" as @@ -360,21 +359,19 @@ def mask_passwords_inplace(object_to_be_masked): .format(f_data)) mask = _dict_update(mask, f_data, recursive_update=True, merge_lists=True) - - mask_by = mask.get('mask_by','******') - + log.debug("Using the mask: {}".format(mask)) + mask_by = mask.get('mask_by', '******') for blacklisted_string in mask.get("blacklisted_strings", []): query_name = blacklisted_string['query_name'] column = blacklisted_string['column'] if query_name != '*': for r in object_to_be_masked: - for query_result in r.get(query_name,{'data':[]})['data']: + for query_result in r.get(query_name, {'data':[]})['data']: if column not in query_result or not isinstance(query_result[column], basestring): # if the column in not present in one data-object, it will # not be present in others as well. Break in that case. # This will happen only if mask.yaml is malformed - print("string", query_name, query_result, column) break value = query_result[column] for pattern in blacklisted_string['blacklisted_patterns']: @@ -385,7 +382,6 @@ def mask_passwords_inplace(object_to_be_masked): for query_name, query_ret in r.iteritems(): for query_result in query_ret['data']: if column not in query_result or not isinstance(query_result[column], basestring): - print("string", query_name, query_result, column) break value = query_result[column] for pattern in blacklisted_string['blacklisted_patterns']: @@ -398,10 +394,9 @@ def mask_passwords_inplace(object_to_be_masked): column = blacklisted_object['column'] if query_name != '*': for r in object_to_be_masked: - for query_result in r.get(query_name,{'data':[]})['data']: + for query_result in r.get(query_name, {'data':[]})['data']: if column not in query_result or \ - ( isinstance(query_result[column], basestring) and query_result[column].strip() != '' ): - print("object", query_name, query_result, column) + (isinstance(query_result[column], basestring) and query_result[column].strip() != '' ): break _recursively_mask_objects(query_result[column], blacklisted_object, mask_by) else: @@ -409,18 +404,15 @@ def mask_passwords_inplace(object_to_be_masked): for query_name, query_ret in r.iteritems(): for query_result in query_ret['data']: if column not in query_result or \ - ( isinstance(query_result[column], basestring) and query_result[column].strip() != '' ): - print("object", query_name, query_result, column) + (isinstance(query_result[column], basestring) and query_result[column].strip() != '' ): break _recursively_mask_objects(query_result[column], blacklisted_object, mask_by) - # successfully masked the object. No need to return anything except Exception as e: log.exception("An error occured while masking the passwords: {}".format(e)) - def _recursively_mask_objects(object_to_mask, blacklisted_object, mask_by): ''' This function is used by "mask_passwords_inplace" to mask passwords contained in From ebfe5047c4150c07ed9e9d5dbda1b8cd894e6e59 Mon Sep 17 00:00:00 2001 From: Yashvinder Singh Date: Tue, 31 Jul 2018 14:39:25 +0530 Subject: [PATCH 12/23] Update nebula_osquery.py --- hubblestack/extmods/modules/nebula_osquery.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 3299b4c1a..72206cd2c 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -56,6 +56,7 @@ def queries(query_group, query_file=None, verbose=False, report_version_with_day=True, + topfile_for_mask=None, mask_passwords=False): ''' Run the set of queries represented by ``query_group`` from the @@ -71,6 +72,10 @@ def queries(query_group, Defaults to False. If set to True, more information (such as the query which was run) will be included in the result. + topfile_for_mask + This is the location of the top file from which the masking information + will be extracted. + mask_passwords Defaults to False. If set to True, passwords mentioned in the return object are masked. @@ -232,7 +237,7 @@ def queries(query_group, result[key] = json.loads(value[len('__JSONIFY__'):]) if mask_passwords: - mask_passwords_inplace(ret) + mask_passwords_inplace(ret, topfile_for_mask) return ret @@ -280,6 +285,7 @@ def hubble_versions(): def top(query_group, topfile='salt://hubblestack_nebula_v2/top.nebula', + topfile_for_mask=None, verbose=False, report_version_with_day=True, mask_passwords=False): @@ -296,6 +302,7 @@ def top(query_group, query_file=configs, verbose=False, report_version_with_day=True, + topfile_for_mask=topfile_for_mask, mask_passwords=mask_passwords) @@ -327,16 +334,20 @@ def get_top_data(topfile): return ret -def mask_passwords_inplace(object_to_be_masked): +def mask_passwords_inplace(object_to_be_masked, topfile): ''' - It masks the passwords present in 'object_to_be_masked'. Uses "mask.yaml" as - a reference to find out the list of blacklisted strings or objects. + It masks the passwords present in 'object_to_be_masked'. Uses mask configuration + file as a reference to find out the list of blacklisted strings or objects. Note that this method alters "object_to_be_masked". + + The path to the mask configuration file can be specified in the "topfile" + argument. ''' try: mask = {} - topfile = 'salt://hubblestack_nebula_v2/top.mask' + if topfile is None: + topfile = 'salt://hubblestack_nebula_v2/top.mask' mask_files = get_top_data(topfile) mask_files = ['salt://hubblestack_nebula_v2/' + mask_file.replace('.', '/') + '.yaml' for mask_file in mask_files] From 995dd6616d3c8113cbd33e382b7390fbb45712d8 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 6 Aug 2018 16:31:41 -0600 Subject: [PATCH 13/23] Add base sphinx files --- .gitignore | 2 + doc/Makefile | 20 ++++ doc/conf.py | 167 ++++++++++++++++++++++++++++++++ doc/contents/gettingstarted.rst | 2 + doc/index.rst | 15 +++ 5 files changed, 206 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/conf.py create mode 100644 doc/contents/gettingstarted.rst create mode 100644 doc/index.rst diff --git a/.gitignore b/.gitignore index e45d16f32..28783696c 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,5 @@ ENV/ # Visual Studio Code settings .vscode + +doc/_build diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 000000000..46c9d3f46 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = HubbleStack +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 000000000..040e7aa11 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +# +# HubbleStack documentation build configuration file, created by +# sphinx-quickstart on Mon Aug 6 15:46:43 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +import os +import sys +sys.path.insert(0, os.path.abspath('../')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'HubbleStack' +copyright = u'2018, Colton Myers, Christer Edwards' +author = u'Colton Myers, Christer Edwards' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'2.4.1' +# The full version, including alpha/beta/rc tags. +release = u'2.4.1-1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'HubbleStackdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'HubbleStack.tex', u'HubbleStack Documentation', + u'Colton Myers, Christer Edwards', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'hubblestack', u'HubbleStack Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'HubbleStack', u'HubbleStack Documentation', + author, 'HubbleStack', 'One line description of project.', + 'Miscellaneous'), +] + + diff --git a/doc/contents/gettingstarted.rst b/doc/contents/gettingstarted.rst new file mode 100644 index 000000000..d45037f69 --- /dev/null +++ b/doc/contents/gettingstarted.rst @@ -0,0 +1,2 @@ +Getting Started with HubbleStack +================================ diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 000000000..35c96b516 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,15 @@ +Welcome to HubbleStack's documentation! +======================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + contents/gettingstarted + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From cedebd1125646038c6dfab16b31a76046c003e1c Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 11:52:36 -0600 Subject: [PATCH 14/23] Add gettingstarted and fill out index --- doc/contents/gettingstarted.rst | 84 +++++++++++++++++++++++++++++++++ doc/index.rst | 12 ++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/doc/contents/gettingstarted.rst b/doc/contents/gettingstarted.rst index d45037f69..5b6c32162 100644 --- a/doc/contents/gettingstarted.rst +++ b/doc/contents/gettingstarted.rst @@ -1,2 +1,86 @@ Getting Started with HubbleStack ================================ + +Installation +------------ + +.. _install_packages: + +Installation Using Released Packages (Recommended) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Various pre-built packages targeting several popular operating systems can be +found under `Releases `_. + +Alternative Installations and Packaging +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Building Hubble packages through Dockerfile +""""""""""""""""""""""""""""""""""""""""""" + +Dockerfile aims to build the Hubble v2 packages easier. Dockerfiles for the +distribution you want to build can be found at the path ``/pkg``. For example, +dockerfile for centos6 distribution is at the path ``/pkg/centos6/`` + +To build an image:: + + docker build -t + +To run the container (which will output the package file in your current +directory):: + + docker run -it --rm -v `pwd`:/data + +Installing using setup.py +""""""""""""""""""""""""" + +.. code-block:: + + sudo yum install git python-setuptools -y + git clone https://github.com/hubblestack/hubble + cd hubble + sudo python setup.py install + +If there are errors installing, it may mean that your setuptools is out of +date. Try this:: + + easy_install pip + pip install -U setuptools + +``setup.py`` installs a hubble "binary" into ``/usr/bin/``. + +A config template has been placed in ``/etc/hubble/hubble``. Modify it to your +specifications and needs. You can do ``hubble -h`` to see the available runtime +options. + +The first two commands you should run to make sure things are set up correctly +are ``hubble --version`` and ``hubble test.ping``. + + +Basic Usage +----------- + +Hubble runs as a standalone agent on each server you wish to monitor. To get +started, install Hubble using one of the above installation options. Once +Hubble is installed, check that everything is working correctly: + +#. Run ``hubble test.ping``. This should return true. +#. Run ``hubble hubble.audit``. You should see the results of the default audit + profiles run against the box + +Quickstart via Docker container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Get up and running with any supported distribution by installing net-tools in a +running docker container. ``docker run -it {distro:tag} sh`` the desired +agent, then use the appropriate package manager to install net-tools: + +To run centos:7 container:: + + docker run -it centos:7 sh + +To install net-tools:: + + yum install net-tools + +Follow instructions above in :ref:`install_packages`. diff --git a/doc/index.rst b/doc/index.rst index 35c96b516..aac1e0335 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,5 +1,13 @@ -Welcome to HubbleStack's documentation! -======================================= +HubbleStack +=========== + +Hubble is a modular, open-source, security & compliance auditing framework +which is built in python, using SaltStack as a library. Hubble provides +on-demand profile-based auditing, real-time security event notifications, +alerting and reporting. It also reports the security information to Splunk, +Logstash, or other endpoints. HubbleStack is a free and open source +project made possible by Adobe. + .. toctree:: :maxdepth: 2 From 321d1a32406a15bc5627f623fd3d0f474952773a Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 13:32:23 -0600 Subject: [PATCH 15/23] More updates + theme --- doc/_static/README | 1 + doc/conf.py | 13 +++++------- doc/contents/gettingstarted.rst | 2 +- doc/index.rst | 7 ++++--- doc/make.bat | 36 +++++++++++++++++++++++++++++++++ doc/requirements.txt | 3 +++ 6 files changed, 50 insertions(+), 12 deletions(-) create mode 100644 doc/_static/README create mode 100644 doc/make.bat create mode 100644 doc/requirements.txt diff --git a/doc/_static/README b/doc/_static/README new file mode 100644 index 000000000..fb78b0cea --- /dev/null +++ b/doc/_static/README @@ -0,0 +1 @@ +This is a placeholder for happier version control diff --git a/doc/conf.py b/doc/conf.py index 040e7aa11..e1277910d 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -68,7 +68,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'README'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' @@ -82,13 +82,10 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} +html_theme = 'sphinx_rtd_theme' +html_theme_path = ['_themes', ] +html_theme_options = { +} # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/doc/contents/gettingstarted.rst b/doc/contents/gettingstarted.rst index 5b6c32162..34dd9c107 100644 --- a/doc/contents/gettingstarted.rst +++ b/doc/contents/gettingstarted.rst @@ -34,7 +34,7 @@ directory):: Installing using setup.py """"""""""""""""""""""""" -.. code-block:: +:: sudo yum install git python-setuptools -y git clone https://github.com/hubblestack/hubble diff --git a/doc/index.rst b/doc/index.rst index aac1e0335..03d63327d 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -8,15 +8,16 @@ alerting and reporting. It also reports the security information to Splunk, Logstash, or other endpoints. HubbleStack is a free and open source project made possible by Adobe. +Table of Contents: +^^^^^^^^^^^^^^^^^^ .. toctree:: :maxdepth: 2 - :caption: Contents: contents/gettingstarted -Indices and tables -================== +Indices and Tables: +^^^^^^^^^^^^^^^^^^^ * :ref:`genindex` * :ref:`modindex` diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 000000000..8423f0996 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=HubbleStack + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 000000000..04c857701 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,3 @@ +sphinx +sphinx-autobuild +sphinx_rtd_theme From 92d013dbddff20cbf9906800d3db9ef93c9e3882 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 14:11:24 -0600 Subject: [PATCH 16/23] Remove all the maintainer blocks --- hubblestack/extmods/grains/cloud_details.py | 4 ---- hubblestack/extmods/grains/custom_grains_pillar.py | 4 ---- hubblestack/extmods/modules/hubble.py | 11 ++--------- hubblestack/extmods/modules/nebula_osquery.py | 5 ----- .../extmods/returners/graylog_nebula_return.py | 4 ---- hubblestack/extmods/returners/graylog_nova_return.py | 4 ---- .../extmods/returners/graylog_pulsar_return.py | 4 ---- .../extmods/returners/logstash_nebula_return.py | 4 ---- hubblestack/extmods/returners/logstash_nova_return.py | 4 ---- .../extmods/returners/logstash_pulsar_return.py | 4 ---- hubblestack/extmods/returners/splunk_nebula_return.py | 4 ---- hubblestack/extmods/returners/splunk_nova_return.py | 4 ---- hubblestack/extmods/returners/splunk_pulsar_return.py | 4 ---- hubblestack/files/hubblestack_nova/command.py | 5 ----- hubblestack/files/hubblestack_nova/cve_scan.py | 6 ------ hubblestack/files/hubblestack_nova/cve_scan_v2.py | 5 ----- hubblestack/files/hubblestack_nova/firewall.py | 5 ----- hubblestack/files/hubblestack_nova/grep.py | 5 ----- hubblestack/files/hubblestack_nova/misc.py | 5 ----- hubblestack/files/hubblestack_nova/mount.py | 5 ----- hubblestack/files/hubblestack_nova/netstat.py | 5 ----- hubblestack/files/hubblestack_nova/openssl.py | 5 ----- hubblestack/files/hubblestack_nova/pkg.py | 5 ----- hubblestack/files/hubblestack_nova/pkgng_audit.py | 6 ------ hubblestack/files/hubblestack_nova/service.py | 5 ----- hubblestack/files/hubblestack_nova/stat_nova.py | 5 ----- hubblestack/files/hubblestack_nova/sysctl.py | 5 ----- hubblestack/files/hubblestack_nova/systemctl.py | 5 ----- hubblestack/files/hubblestack_nova/vulners_scanner.py | 5 ----- hubblestack/files/hubblestack_nova/win_auditpol.py | 7 +------ hubblestack/files/hubblestack_nova/win_firewall.py | 6 ------ hubblestack/files/hubblestack_nova/win_gp.py | 6 ------ hubblestack/files/hubblestack_nova/win_pkg.py | 7 +------ hubblestack/files/hubblestack_nova/win_reg.py | 6 ------ hubblestack/files/hubblestack_nova/win_secedit.py | 7 +------ 35 files changed, 5 insertions(+), 176 deletions(-) diff --git a/hubblestack/extmods/grains/cloud_details.py b/hubblestack/extmods/grains/cloud_details.py index b9ae742ba..4a4e53be2 100644 --- a/hubblestack/extmods/grains/cloud_details.py +++ b/hubblestack/extmods/grains/cloud_details.py @@ -1,9 +1,5 @@ ''' HubbleStack Cloud Details Grain - -:maintainer: HubbleStack -:platform: All -:requires: SaltStack ''' import requests diff --git a/hubblestack/extmods/grains/custom_grains_pillar.py b/hubblestack/extmods/grains/custom_grains_pillar.py index fd2436edb..15cd700b9 100644 --- a/hubblestack/extmods/grains/custom_grains_pillar.py +++ b/hubblestack/extmods/grains/custom_grains_pillar.py @@ -3,10 +3,6 @@ Allows for fetching custom grain and pillar data from a local salt-minion via salt-call - -:maintainer: HubbleStack -:platform: All -:requires: SaltStack ''' import re diff --git a/hubblestack/extmods/modules/hubble.py b/hubblestack/extmods/modules/hubble.py index fd25d15f6..78261c135 100644 --- a/hubblestack/extmods/modules/hubble.py +++ b/hubblestack/extmods/modules/hubble.py @@ -2,11 +2,6 @@ ''' Loader and primary interface for nova modules -:maintainer: HubbleStack / basepi -:maturity: 2016.10.2 -:platform: All -:requires: SaltStack - See README for documentation Configuration: @@ -33,7 +28,7 @@ import salt.utils from salt.exceptions import CommandExecutionError from hubblestack import __version__ -from nova_loader import NovaLazyLoader +from hubblestack.extmods.modules.nova_loader import NovaLazyLoader __nova__ = {} @@ -104,9 +99,7 @@ def audit(configs=None, Any parameters & values that are not explicitly defined will be passed directly through to the Nova module(s). - CLI Examples: - - .. code-block:: bash + CLI Examples:: salt '*' hubble.audit foo salt '*' hubble.audit foo,bar tags='CIS*' diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 895c98b7c..0511275d2 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -2,11 +2,6 @@ ''' osquery wrapper for HubbleStack Nebula -:maintainer: basepi -:maturity: 2016.10.2 -:platform: All -:requires: SaltStack, osquery - Designed to run sets of osquery queries defined in pillar. These sets will have a unique identifier, and be targeted by identifier. Usually, this identifier will be a frequency. ('15 minutes', '1 day', etc). Identifiers are diff --git a/hubblestack/extmods/returners/graylog_nebula_return.py b/hubblestack/extmods/returners/graylog_nebula_return.py index a650f7934..978502ad6 100644 --- a/hubblestack/extmods/returners/graylog_nebula_return.py +++ b/hubblestack/extmods/returners/graylog_nebula_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Nebula-to-graylog (http input) returner -:maintainer: HubbleStack -:platform: All -:requires: HubbleStack - Deliver HubbleStack Nebula query data into graylog using the HTTP input plugin. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/graylog_nova_return.py b/hubblestack/extmods/returners/graylog_nova_return.py index 34d994942..f01d6546a 100644 --- a/hubblestack/extmods/returners/graylog_nova_return.py +++ b/hubblestack/extmods/returners/graylog_nova_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Nova-to-graylog (http input) returner -:maintainer: HubbleStack -:platform: All -:requires: HubbleStack - Deliver HubbleStack Nova data into graylog using the HTTP input plugin. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/graylog_pulsar_return.py b/hubblestack/extmods/returners/graylog_pulsar_return.py index f75f09a87..5f77529e9 100644 --- a/hubblestack/extmods/returners/graylog_pulsar_return.py +++ b/hubblestack/extmods/returners/graylog_pulsar_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Pulsar-to-graylog (http input) returner -:maintainer: HubbleStack -:platform: All -:requires: HubbleStack - Deliver HubbleStack Pulsar event data into graylog using the HTTP input plugin. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/logstash_nebula_return.py b/hubblestack/extmods/returners/logstash_nebula_return.py index 22070a7f5..36d3925cd 100644 --- a/hubblestack/extmods/returners/logstash_nebula_return.py +++ b/hubblestack/extmods/returners/logstash_nebula_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Nebula-to-Logstash (http input) returner -:maintainer: HubbleStack -:platform: All -:requires: HubbleStack - Deliver HubbleStack Nebula query data into Logstash using the HTTP input plugin. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/logstash_nova_return.py b/hubblestack/extmods/returners/logstash_nova_return.py index b48bc824e..a7f3e3817 100644 --- a/hubblestack/extmods/returners/logstash_nova_return.py +++ b/hubblestack/extmods/returners/logstash_nova_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Nova-to-Logstash (http input) returner -:maintainer: HubbleStack -:platform: All -:requires: HubbleStack - Deliver HubbleStack Nova data into Logstash using the HTTP input plugin. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/logstash_pulsar_return.py b/hubblestack/extmods/returners/logstash_pulsar_return.py index e29fd120e..08bb1d316 100644 --- a/hubblestack/extmods/returners/logstash_pulsar_return.py +++ b/hubblestack/extmods/returners/logstash_pulsar_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Pulsar-to-Logstash (http input) returner -:maintainer: HubbleStack -:platform: All -:requires: HubbleStack - Deliver HubbleStack Pulsar event data into Logstash using the HTTP input plugin. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/splunk_nebula_return.py b/hubblestack/extmods/returners/splunk_nebula_return.py index 5d1359459..35897616c 100644 --- a/hubblestack/extmods/returners/splunk_nebula_return.py +++ b/hubblestack/extmods/returners/splunk_nebula_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Nebula-to-Splunk returner -:maintainer: HubbleStack -:platform: All -:requires: SaltStack - Deliver HubbleStack Nebula query data into Splunk using the HTTP event collector. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/splunk_nova_return.py b/hubblestack/extmods/returners/splunk_nova_return.py index c845cead7..040c0880d 100644 --- a/hubblestack/extmods/returners/splunk_nova_return.py +++ b/hubblestack/extmods/returners/splunk_nova_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Nova-to-Splunk returner -:maintainer: HubbleStack -:platform: All -:requires: SaltStack - Deliver HubbleStack Nova result data into Splunk using the HTTP event collector. Required config/pillar settings: diff --git a/hubblestack/extmods/returners/splunk_pulsar_return.py b/hubblestack/extmods/returners/splunk_pulsar_return.py index 983c05402..d50c7d19e 100644 --- a/hubblestack/extmods/returners/splunk_pulsar_return.py +++ b/hubblestack/extmods/returners/splunk_pulsar_return.py @@ -2,10 +2,6 @@ ''' HubbleStack Pulsar-to-Splunk returner -:maintainer: HubbleStack -:platform: All -:requires: SaltStack - Deliver HubbleStack Pulsar event data into Splunk using the HTTP event collector. Required config/pillar settings: diff --git a/hubblestack/files/hubblestack_nova/command.py b/hubblestack/files/hubblestack_nova/command.py index d44cd18e3..061a9cec1 100644 --- a/hubblestack/files/hubblestack_nova/command.py +++ b/hubblestack/files/hubblestack_nova/command.py @@ -8,11 +8,6 @@ to enable this module). This allows nova to run arbitrary commands via yaml profiles. -:maintainer: HubbleStack / basepi -:maturity: 2016.7.0 -:platform: All -:requires: SaltStack - Sample YAML data, with inline comments: # Top level key lets the module know it should look at this data diff --git a/hubblestack/files/hubblestack_nova/cve_scan.py b/hubblestack/files/hubblestack_nova/cve_scan.py index 1133152cb..89fa3e4e9 100644 --- a/hubblestack/files/hubblestack_nova/cve_scan.py +++ b/hubblestack/files/hubblestack_nova/cve_scan.py @@ -1,12 +1,6 @@ # -*- encoding: utf-8 -*- ''' HubbleStack Nova plugin for openscap scanning. - -:maintainer: HubbleStack / cedwards -:maturity: 2016.7.0 -:platform: Red Hat -:requires: SaltStack + oscap execution module - ''' from __future__ import absolute_import import salt.utils diff --git a/hubblestack/files/hubblestack_nova/cve_scan_v2.py b/hubblestack/files/hubblestack_nova/cve_scan_v2.py index a51555dac..83bd4550e 100644 --- a/hubblestack/files/hubblestack_nova/cve_scan_v2.py +++ b/hubblestack/files/hubblestack_nova/cve_scan_v2.py @@ -7,11 +7,6 @@ the yaml profile, and that data cached at the path /var/cache/salt/minion/cve_scan_cache/_.json -:maintainer: HubbleStack / jaredhanson11 -:maturity: 2016.7.0 -:platform: Linux -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'cve_scan_v2' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/firewall.py b/hubblestack/files/hubblestack_nova/firewall.py index d9adaa62e..bc39f3cd0 100644 --- a/hubblestack/files/hubblestack_nova/firewall.py +++ b/hubblestack/files/hubblestack_nova/firewall.py @@ -2,11 +2,6 @@ ''' Hubble Nova plugin for using iptables to verify firewall rules -:maintainer: HubbleStack / avb76 -:maturity: 2016.7.0 -:platform: Linux -:requires: SaltStack - This audit module requires yaml data to execute. Running hubble.audit will search the local directory for any .yaml files and it will pass all the data to this module. If this module find a top-level 'firewall' key, it will use the diff --git a/hubblestack/files/hubblestack_nova/grep.py b/hubblestack/files/hubblestack_nova/grep.py index 6de3d10f3..be46cc77f 100644 --- a/hubblestack/files/hubblestack_nova/grep.py +++ b/hubblestack/files/hubblestack_nova/grep.py @@ -6,11 +6,6 @@ not be found in the specified file. Whitelisted patterns must be found in the specified file. -:maintainer: HubbleStack / basepi -:maturity: 2016.7.0 -:platform: All -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'grep' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/misc.py b/hubblestack/files/hubblestack_nova/misc.py index 3c64038fa..e174c923a 100644 --- a/hubblestack/files/hubblestack_nova/misc.py +++ b/hubblestack/files/hubblestack_nova/misc.py @@ -4,11 +4,6 @@ run more complex nova audits without allowing arbitrary command execution from within the yaml profiles. -:maintainer: HubbleStack / basepi -:maturity: 2016.7.2 -:platform: All -:requires: SaltStack - Sample YAML data, with inline comments: # Top level key lets the module know it should look at this data diff --git a/hubblestack/files/hubblestack_nova/mount.py b/hubblestack/files/hubblestack_nova/mount.py index 456d952de..fa92fab81 100644 --- a/hubblestack/files/hubblestack_nova/mount.py +++ b/hubblestack/files/hubblestack_nova/mount.py @@ -6,11 +6,6 @@ not be found in the specified file. Whitelisted patterns must be found in the specified file. -:maintainer: HubbleStack / basepi -:maturity: 2017.8.29 -:platform: All -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'mount' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/netstat.py b/hubblestack/files/hubblestack_nova/netstat.py index 36dafb8c3..3cad4ae02 100644 --- a/hubblestack/files/hubblestack_nova/netstat.py +++ b/hubblestack/files/hubblestack_nova/netstat.py @@ -2,11 +2,6 @@ ''' HubbleStack Nova module for auditing open ports. -:maintainer: HubbleStack / basepi -:maturity: 2016.7.0 -:platform: Unix -:requires: SaltStack - Sample data for the netstat whitelist: .. code-block:: yaml diff --git a/hubblestack/files/hubblestack_nova/openssl.py b/hubblestack/files/hubblestack_nova/openssl.py index 589734331..1c16fb7b7 100644 --- a/hubblestack/files/hubblestack_nova/openssl.py +++ b/hubblestack/files/hubblestack_nova/openssl.py @@ -2,11 +2,6 @@ ''' HubbleStack Nova module for auditing SSL certificates. -:maintainer: HubbleStack / avb76 -:maturity: 2016.7.0 -:platform: Linux -:requires: SaltStack, python-OpenSSL - This audit module requires YAML data to execute. It will search the yaml data received for the topkey 'openssl'. diff --git a/hubblestack/files/hubblestack_nova/pkg.py b/hubblestack/files/hubblestack_nova/pkg.py index ef2bbc675..de62ac1be 100644 --- a/hubblestack/files/hubblestack_nova/pkg.py +++ b/hubblestack/files/hubblestack_nova/pkg.py @@ -6,11 +6,6 @@ must not be installed. Whitelisted packages must be installed, with options for requiring a specific version or a minimum or maximum version. -:maintainer: HubbleStack / basepi -:maturity: 2016.7.0 -:platform: All -:requires: SaltStack - Sample YAML data, with inline comments: pkg: diff --git a/hubblestack/files/hubblestack_nova/pkgng_audit.py b/hubblestack/files/hubblestack_nova/pkgng_audit.py index afe91cd92..086c4c8ae 100644 --- a/hubblestack/files/hubblestack_nova/pkgng_audit.py +++ b/hubblestack/files/hubblestack_nova/pkgng_audit.py @@ -1,12 +1,6 @@ # -*- encoding: utf-8 -*- ''' Hubble Nova plugin for FreeBSD pkgng audit - -:maintainer: HubbleStack / cedwards -:maturity: 2016.7.0 -:platform: FreeBSD -:requires: SaltStack - ''' from __future__ import absolute_import import logging diff --git a/hubblestack/files/hubblestack_nova/service.py b/hubblestack/files/hubblestack_nova/service.py index 227f4be47..b1758a5fc 100644 --- a/hubblestack/files/hubblestack_nova/service.py +++ b/hubblestack/files/hubblestack_nova/service.py @@ -5,11 +5,6 @@ Supports both blacklisting and whitelisting services. Blacklisted services must not be running. Whitelisted services must be running. -:maintainer: HubbleStack / basepi -:maturity: 2016.7.0 -:platform: All -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'service' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/stat_nova.py b/hubblestack/files/hubblestack_nova/stat_nova.py index dc4043fbb..7e35ee3ba 100644 --- a/hubblestack/files/hubblestack_nova/stat_nova.py +++ b/hubblestack/files/hubblestack_nova/stat_nova.py @@ -2,11 +2,6 @@ ''' HubbleStack Nova module for using stat to verify ownership & permissions. -:maintainer: HubbleStack / avb76 -:maturity: 2016.7.0 -:platform: Linux -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'stat' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/sysctl.py b/hubblestack/files/hubblestack_nova/sysctl.py index 9db476f88..b268ad22e 100644 --- a/hubblestack/files/hubblestack_nova/sysctl.py +++ b/hubblestack/files/hubblestack_nova/sysctl.py @@ -2,11 +2,6 @@ ''' HubbleStack Nova module for using sysctl to verify sysctl parameter. -:maintainer: HubbleStack / avb76 -:maturity: 2016.7.0 -:platform: Linux -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'sysctl' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/systemctl.py b/hubblestack/files/hubblestack_nova/systemctl.py index ce12b02e9..dfc3e2344 100644 --- a/hubblestack/files/hubblestack_nova/systemctl.py +++ b/hubblestack/files/hubblestack_nova/systemctl.py @@ -5,11 +5,6 @@ Supports both blacklisting and whitelisting patterns. Blacklisted services must not be enabled. Whitelisted services must be enabled. -:maintainer: HubbleStack / basepi -:maturity: 2017.8.29 -:platform: All -:requires: SaltStack - This audit module requires yaml data to execute. It will search the local directory for any .yaml files, and if it finds a top-level 'systemctl' key, it will use that data. diff --git a/hubblestack/files/hubblestack_nova/vulners_scanner.py b/hubblestack/files/hubblestack_nova/vulners_scanner.py index 7db214944..1c288d6dc 100644 --- a/hubblestack/files/hubblestack_nova/vulners_scanner.py +++ b/hubblestack/files/hubblestack_nova/vulners_scanner.py @@ -7,11 +7,6 @@ The API is described at the link below: https://blog.vulners.com/linux-vulnerability-audit-in-vulners/ -:maintainer: HubbleStack / avb76 -:maturity: 3/26/2017 (TODO: change the format when the release date is established) -:platform: Linux -:requires: SaltStack - This audit module requires a YAML file inside the hubblestack_nova_profiles directory. The file should have the following format: diff --git a/hubblestack/files/hubblestack_nova/win_auditpol.py b/hubblestack/files/hubblestack_nova/win_auditpol.py index 2c8069979..b46c4a6c9 100644 --- a/hubblestack/files/hubblestack_nova/win_auditpol.py +++ b/hubblestack/files/hubblestack_nova/win_auditpol.py @@ -1,11 +1,6 @@ # -*- encoding: utf-8 -*- ''' - -:maintainer: HubbleStack / madchills -:maturity: 2016.7.0 -:platform: Windows -:requires: SaltStack - +Windows auditpol audit module ''' from __future__ import absolute_import diff --git a/hubblestack/files/hubblestack_nova/win_firewall.py b/hubblestack/files/hubblestack_nova/win_firewall.py index b384c9e3f..13278b5df 100644 --- a/hubblestack/files/hubblestack_nova/win_firewall.py +++ b/hubblestack/files/hubblestack_nova/win_firewall.py @@ -1,12 +1,6 @@ # -*- encoding: utf-8 -*- ''' HubbleStack Nova Windows Firewall module - -:maintainer: HubbleStack / madchills -:maturity: 2016.7.0 -:platform: Windows -:requires: SaltStack - ''' from __future__ import absolute_import diff --git a/hubblestack/files/hubblestack_nova/win_gp.py b/hubblestack/files/hubblestack_nova/win_gp.py index eb948b645..5b5586227 100644 --- a/hubblestack/files/hubblestack_nova/win_gp.py +++ b/hubblestack/files/hubblestack_nova/win_gp.py @@ -1,11 +1,5 @@ # -*- encoding: utf-8 -*- ''' - -:maintainer: HubbleStack -:maturity: 2016.7.0 -:platform: Windows -:requires: SaltStack - ''' from __future__ import absolute_import diff --git a/hubblestack/files/hubblestack_nova/win_pkg.py b/hubblestack/files/hubblestack_nova/win_pkg.py index bb6b7d9e6..e3ec5a08b 100644 --- a/hubblestack/files/hubblestack_nova/win_pkg.py +++ b/hubblestack/files/hubblestack_nova/win_pkg.py @@ -1,11 +1,6 @@ # -*- encoding: utf-8 -*- ''' - -:maintainer: HubbleStack -:maturity: 2016.7.0 -:platform: Windows -:requires: SaltStack - +Windows package audit module ''' from __future__ import absolute_import diff --git a/hubblestack/files/hubblestack_nova/win_reg.py b/hubblestack/files/hubblestack_nova/win_reg.py index 3e9d1aa0d..1299e31fb 100644 --- a/hubblestack/files/hubblestack_nova/win_reg.py +++ b/hubblestack/files/hubblestack_nova/win_reg.py @@ -1,11 +1,5 @@ # -*- encoding: utf-8 -*- ''' - -:maintainer: HubbleStack / madchills -:maturity: 2016.7.0 -:platform: Windows -:requires: SaltStack - ''' from __future__ import absolute_import diff --git a/hubblestack/files/hubblestack_nova/win_secedit.py b/hubblestack/files/hubblestack_nova/win_secedit.py index f222735a5..e90fbfabe 100644 --- a/hubblestack/files/hubblestack_nova/win_secedit.py +++ b/hubblestack/files/hubblestack_nova/win_secedit.py @@ -1,11 +1,6 @@ # -*- encoding: utf-8 -*- ''' - -:maintainer: HubbleStack / madchills -:maturity: 2016.7.0 -:platform: Windows -:requires: SaltStack - +Windows secedit audit module ''' from __future__ import absolute_import From 114ca795d0de45ecad384f1a156dd0a3a6a8979e Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 14:22:17 -0600 Subject: [PATCH 17/23] Add module documentation and nova documentation --- doc/conf.py | 9 +- doc/contents/auditing_nova.rst | 118 ++++++++++++++++++++++++ doc/contents/module_documentation.rst | 9 ++ doc/contents/modules/nebula_osquery.rst | 5 + doc/contents/modules/nova.rst | 5 + doc/contents/modules/pulsar.rst | 5 + doc/contents/modules/win_pulsar.rst | 5 + doc/index.rst | 12 ++- 8 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 doc/contents/auditing_nova.rst create mode 100644 doc/contents/module_documentation.rst create mode 100644 doc/contents/modules/nebula_osquery.rst create mode 100644 doc/contents/modules/nova.rst create mode 100644 doc/contents/modules/pulsar.rst create mode 100644 doc/contents/modules/win_pulsar.rst diff --git a/doc/conf.py b/doc/conf.py index e1277910d..581a31ab6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -29,8 +29,13 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.viewcode'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', +] + +todo_include_todos = True # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/doc/contents/auditing_nova.rst b/doc/contents/auditing_nova.rst new file mode 100644 index 000000000..495f32eaf --- /dev/null +++ b/doc/contents/auditing_nova.rst @@ -0,0 +1,118 @@ +Auditing (Nova) +=============== + +Hubble supports success/fail auditing via a number of included modules. The +codename for the audit piece of hubble is "Nova". + +Usage +----- + +There are two primary entry points for the Nova module: + +``hubble.audit`` + + audits the agent using the YAML profile(s) you provide as comma-separated + arguments. + + ``hubble.audit`` takes a number of optional arguments. The first is a + comma-separated list of paths. These paths can be files or directories + within the ``hubblestack_nova_profiles`` directory, with the ``.yaml`` + suffix removed. For information on the other arguments, please see + :doc:`modules/nova`. + + If ``hubble.audit`` is run without targeting any audit configs or + directories, it will instead run ``hubble.top`` with no arguments. + + ``hubble.audit`` will return a list of audits which were successful, and a + list of audits which failed. + +``hubble.top`` + + audits the agent using the ``top.nova`` configuration. By default, the + ``top.nova`` should be located in the fileserver at + ``salt://hubblestack_nova_profiles/top.nova``, but a different path can be + defined. + +Here are some example calls for ``hubble.audit``:: + + # Run the cve scanner and the CIS profile: + hubble hubble.audit cve.scan-v2,cis.centos-7-level-1-scored-v1 + # Run hubble.top with the default topfile (top.nova) + hubble hubble.top + # Run all yaml configs and tags under salt://hubblestack_nova_profiles/foo/ and salt://hubblestack_nova_profiles/bar, but only run audits with tags starting with "CIS" + hubble hubble.audit foo,bar tags='CIS*' + +Configuration +------------- + +For Nova module, configurations can be done via Nova topfiles. Nova topfiles +look very similar to saltstack topfiles, except the top-level key is always +nova, as nova doesn’t have environments. + +**hubblestack_data/hubblestack_nova_profiles/top.nova**:: + + nova: + '*': + - cve.scan-v2 + - network.ssh + - network.smtp + 'web*': + - cis.centos-7-level-1-scored-v1 + - cis.centos-7-level-2-scored-v1 + 'G@os_family:debian': + - network.ssh + - cis.debian-7-level-1-scored: 'CIS*' + +Additionally, all nova topfile matches are compound matches, so you never need +to define a match type like you do in saltstack topfiles. Each list item is a +string representing the dot-separated location of a yaml file which will be run +with ``hubble.audit``. You can also specify a tag glob to use as a filter for +just that yaml file, using a colon after the yaml file (turning it into a +dictionary). See the last two lines in the yaml above for examples. + +Examples:: + + hubble hubble.top + hubble hubble.top foo/bar/top.nova + hubble hubble.top foo/bar.nova verbose=True + +In some cases, your organization may want to skip certain audit checks for +certain hosts. This is supported via compensating control configuration. + +You can skip a check globally by adding a ``control: `` key to the +check itself. This key should be added at the same level as description and +trigger pieces of a check. In this case, the check will never run, and will +output under the Controlled results key. + +Nova also supports separate control profiles, for more fine-grained control +using topfiles. You can use a separate YAML top-level key called control. +Generally, you’ll put this top-level key inside of a separate YAML file and +only include it in the top-data for the hosts for which it is relevant. + +For these separate control configs, the audits will always run, whether they +are controlled or not. However, controlled audits which fail will be converted +from Failure to Controlled in a post-processing operation. + +The control config syntax is as follows: + +**hubblestack_data/hubblestack_nova_profiles/example_control/example.yaml**:: + + control: + - CIS-2.1.4: This is the reason we control the check + - some_other_tag: + reason: This is the reason we control the check + - a_third_tag_with_no_reason + +Note that providing a reason for the control is optional. Any of the three +formats shown in the yaml list above will work. + +Once you have your compensating control config, just target the yaml to the +hosts you want to control using your topfile. In this case, all the audits will +still run, but if any of the controlled checks fail, they will be removed from +Failure and added to Controlled, and will be treated as a Success for the +purposes of compliance percentage. To use the above control, you would add the +following to your ``top.nova`` file:: + + nova: + '*': + - example_control.example diff --git a/doc/contents/module_documentation.rst b/doc/contents/module_documentation.rst new file mode 100644 index 000000000..c60830aad --- /dev/null +++ b/doc/contents/module_documentation.rst @@ -0,0 +1,9 @@ +Module Documentation +==================== + +.. toctree:: + + modules/nova + modules/nebula_osquery + modules/pulsar + modules/win_pulsar diff --git a/doc/contents/modules/nebula_osquery.rst b/doc/contents/modules/nebula_osquery.rst new file mode 100644 index 000000000..b3697e688 --- /dev/null +++ b/doc/contents/modules/nebula_osquery.rst @@ -0,0 +1,5 @@ +Nebula (``nebula_osquery.py``) +============================== + +.. automodule:: hubblestack.extmods.modules.nebula_osquery + :members: queries, top diff --git a/doc/contents/modules/nova.rst b/doc/contents/modules/nova.rst new file mode 100644 index 000000000..bc673e525 --- /dev/null +++ b/doc/contents/modules/nova.rst @@ -0,0 +1,5 @@ +Nova (``hubble.py``) +==================== + +.. automodule:: hubblestack.extmods.modules.hubble + :members: audit, top diff --git a/doc/contents/modules/pulsar.rst b/doc/contents/modules/pulsar.rst new file mode 100644 index 000000000..16adf0110 --- /dev/null +++ b/doc/contents/modules/pulsar.rst @@ -0,0 +1,5 @@ +Pulsar (Linux) (``pulsar.py``) +============================== + +.. automodule:: hubblestack.extmods.modules.pulsar + :members: process, top diff --git a/doc/contents/modules/win_pulsar.rst b/doc/contents/modules/win_pulsar.rst new file mode 100644 index 000000000..319154232 --- /dev/null +++ b/doc/contents/modules/win_pulsar.rst @@ -0,0 +1,5 @@ +Pulsar (Windows) (``win_pulsar.py``) +==================================== + +.. automodule:: hubblestack.extmods.modules.win_pulsar + :members: process, top diff --git a/doc/index.rst b/doc/index.rst index 03d63327d..73d74a084 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,11 +1,11 @@ HubbleStack =========== -Hubble is a modular, open-source, security & compliance auditing framework -which is built in python, using SaltStack as a library. Hubble provides -on-demand profile-based auditing, real-time security event notifications, -alerting and reporting. It also reports the security information to Splunk, -Logstash, or other endpoints. HubbleStack is a free and open source +HubbleStack (Hubble for short) is a modular, open-source, security & compliance +auditing framework which is built in python, using SaltStack as a library. +Hubble provides on-demand profile-based auditing, real-time security event +notifications, alerting and reporting. It also reports the security information +to Splunk, Logstash, or other endpoints. HubbleStack is a free and open source project made possible by Adobe. Table of Contents: @@ -15,6 +15,8 @@ Table of Contents: :maxdepth: 2 contents/gettingstarted + contents/auditing_nova + contents/module_documentation Indices and Tables: ^^^^^^^^^^^^^^^^^^^ From aea5cc9b257aba214e73864eee0664606a1d8840 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 14:54:18 -0600 Subject: [PATCH 18/23] Doc fixes for hubblestack modules --- hubblestack/extmods/modules/nebula_osquery.py | 2 +- hubblestack/extmods/modules/win_pulsar.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hubblestack/extmods/modules/nebula_osquery.py b/hubblestack/extmods/modules/nebula_osquery.py index 0511275d2..ab03ed5e1 100644 --- a/hubblestack/extmods/modules/nebula_osquery.py +++ b/hubblestack/extmods/modules/nebula_osquery.py @@ -77,7 +77,7 @@ def queries(query_group, CLI Examples: - .. code_block:: bash + .. code-block:: bash salt '*' nebula.queries day salt '*' nebula.queries hour verbose=True diff --git a/hubblestack/extmods/modules/win_pulsar.py b/hubblestack/extmods/modules/win_pulsar.py index 0d736e24c..ab272bd23 100644 --- a/hubblestack/extmods/modules/win_pulsar.py +++ b/hubblestack/extmods/modules/win_pulsar.py @@ -46,6 +46,7 @@ def process(configfile='salt://hubblestack_pulsar/hubblestack_pulsar_win_config. Example yaml config on fileserver (targeted by configfile option) .. code-block:: yaml + C:\Users: {} C:\Windows: mask: From 2b4665523313762fde23fb95849714902ee81a86 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 15:46:53 -0600 Subject: [PATCH 19/23] Add a bunch more docs, and migrate to an rST readme --- README.md | 323 -------------------------------- README.rst | 8 + doc/contents/auditing_nova.rst | 5 + doc/contents/fim_pulsar.rst | 112 +++++++++++ doc/contents/fim_win_pulsar.rst | 29 +++ doc/contents/insight_nebula.rst | 74 ++++++++ doc/index.rst | 3 + 7 files changed, 231 insertions(+), 323 deletions(-) delete mode 100644 README.md create mode 100644 README.rst create mode 100644 doc/contents/fim_pulsar.rst create mode 100644 doc/contents/fim_win_pulsar.rst create mode 100644 doc/contents/insight_nebula.rst diff --git a/README.md b/README.md deleted file mode 100644 index 68f6b37bc..000000000 --- a/README.md +++ /dev/null @@ -1,323 +0,0 @@ -Table of Contents -================= - - * [HUBBLE](#hubble) - * [Packaging / Installing](#packaging--installing) - * [Installing using setup.py](#installing-using-setuppy) - * [Building Hubble packages through Dockerfile](#building-hubble-packages-through-dockerfile) - * [Using released packages](#using-released-packages) - * [Getting Started](#getting-started) - * [Nova](#nova) - * [Usage](#usage) - * [Configuration](#configuration) - * [Nebula](#nebula) - * [Usage](#usage-1) - * [Configuration](#configuration-1) - * [Pulsar](#pulsar) - * [Usage](#usage-2) - * [Configuration](#configuration-2) - * [Excluding Paths](#excluding-paths) - * [Pulsar topfile top.pulsar](#pulsar-topfile-toppulsar) - -# HUBBLE - -Hubble is a modular, open-source, security & compliance auditing framework which is built on SaltStack. It is alternate version of Hubblestack which can be run without an existing SaltStack infrastructure. Hubble provides on-demand profile-based auditing, real-time security event notifications, alerting and reporting. It also reports the security information to Splunk. This document describes installation, configuration and general use of Hubble. - -## Packaging / Installing -### Installing using setup.py -```sh -sudo yum install git python-setuptools -y -git clone https://github.com/hubblestack/hubble -cd hubble -sudo python setup.py install -``` - -If there are errors installing, it may mean that your setuptools is out of -date. Try this: - -```sh -easy_install pip -pip install -U setuptools -``` - -Installs a hubble "binary" into `/usr/bin/`. - -A config template has been placed in `/etc/hubble/hubble`. Modify it to your specifications and needs. You can do `hubble -h` to see the available options. - -The first two commands you should run to make sure things are set up correctly are `hubble --version` and `hubble test.ping`. - -### Quickstart Docker container -Get up and running with any supported distribution by installing net-tools in a running docker container. -`docker run -it {distro:tag} sh` the desired agent, then use the appropriate package manager to install net-tools: - -To run centos:7 container -```sh -docker run -it centos:7 sh -``` -To install net-tools -```sh -yum install net-tools -``` -Follow instructions above in _Installing using setup.py_ - -### Building Hubble packages through Dockerfile -Dockerfile aims to build the Hubble v2 packages easier. Dockerfiles for the distribution you want to build can be found at the path `/pkg`. For example, dockerfile for centos6 distribution is at the path `/pkg/centos6/` - -To build an image -```sh -docker build -t -``` -To run the container (which will output the package file in your current directory) -```sh -docker run -it --rm -v `pwd`:/data -``` - -### Using released packages -Various pre-built packages targeting several popular operating systems can be found under [Releases](https://github.com/hubblestack/hubble/releases). - -## Getting Started -Hubble runs as a standalone agent on each server you wish to monitor. There are no masters or minions. To get started, install Hubble using one of the above installation options. Once Hubble is installed, check that everything is working correctly: -1. Run `hubble test.ping`. This should return true. -1. Run `hubble hubble.sync`. In a default installation, this will sync the latest version of [hubblestack_data](https://github.com/hubblestack/hubblestack_data) to the agent. -1. Run `hubble hubble.load`. This loads the synced files into Hubble. -1. Run `hubble hubble.audit`. You should see the results of the default audit configuration run against the agent. - -The next step will likely be to switch to your own top file. Create a file `/srv/salt/hubblestack_data/hubblestack_nova_profiles/top.nova` with the contents: -``` -nova: - '*': - - network.ssh -``` -Now, edit the Hubble configuration file `/etc/hubble/hubble`. Uncomment the following lines: -``` -# file_roots: -# base: -# - /srv/salt/hubblestack_data -``` -With the file_root configuration added, run `hubble hubble.audit` (Audit will automatically run both sync and load before performing the audit). The agent should now only run the network.ssh profile. - -From here, you can now modify your top file to suit your requirements, or add your own profiles and controls. Also note that you can move your hubblestack_data directory anywhere you like. You could even create your own git repository and host your files there if you prefer. You would simply add another gitfs_remote to the configuration file. - -## Nova -Nova is Hubble's auditing system. -### Usage -There are four primary functions for Nova module: -- `hubble.sync` : syncs the `hubblestack_nova_profiles/` and `hubblestack_nova/` directories to the host(s). -- `hubble.load` : loads the synced configuration files. -- `hubble.audit` : audits the agent using the YAML profile(s) you provide as comma-separated arguments. hubble.audit takes two optional arguments. The first is a comma-separated list of paths. These paths can be files or directories within the `hubblestack_nova_profiles` directory. The second argument allows for toggling Nova configuration, such as verbosity, level of detail, etc. If `hubble.audit` is run without targeting any audit configs or directories, it will instead run `hubble.top` with no arguments. `hubble.audit` will return a list of audits which were successful, and a list of audits which failed. -- `hubble.top` : audits the agent using the top.nova configuration. - -Here are some example calls for `hubble.audit`: -```sh -# Run the cve scanner and the CIS profile: -hubble hubble.audit cve.scan-v2,cis.centos-7-level-1-scored-v1 -# Run hubble.top with the default topfile (top.nova) -hubble hubble.top -# Run all yaml configs and tags under salt://hubblestack_nova_profiles/foo/ and salt://hubblestack_nova_profiles/bar, but only run audits with tags starting with "CIS" -hubble hubble.audit foo,bar tags='CIS*' -``` -### Configuration -For Nova module, configurations can be done via Nova topfiles. Nova topfiles look very similar to saltstack topfiles, except the top-level key is always nova, as nova doesn’t have environments. - -**hubblestack_data/hubblestack_nova_profiles/top.nova** -```sh -nova: - '*': - - cve.scan-v2 - - network.ssh - - network.smtp - 'web*': - - cis.centos-7-level-1-scored-v1 - - cis.centos-7-level-2-scored-v1 - 'G@os_family:debian': - - network.ssh - - cis.debian-7-level-1-scored: 'CIS*' -``` -Additionally, all nova topfile matches are compound matches, so you never need to define a match type like you do in saltstack topfiles. Each list item is a string representing the dot-separated location of a yaml file which will be run with `hubble.audit`. You can also specify a tag glob to use as a filter for just that yaml file, using a colon after the yaml file (turning it into a dictionary). See the last two lines in the yaml above for examples. - -Examples: -```sh -hubble hubble.top -hubble hubble.top foo/bar/top.nova -hubble hubble.top foo/bar.nova verbose=True -``` - -In some cases, your organization may want to skip certain audit checks for certain hosts. This is supported via compensating control configuration. - -You can skip a check globally by adding a `control: ` key to the check itself. This key should be added at the same level as description and trigger pieces of a check. In this case, the check will never run, and will output under the Controlled results key. - -Nova also supports separate control profiles, for more fine-grained control using topfiles. You can use a separate YAML top-level key called control. Generally, you’ll put this top-level key inside of a separate YAML file and only include it in the top-data for the hosts for which it is relevant. - -For these separate control configs, the audits will always run, whether they are controlled or not. However, controlled audits which fail will be converted from Failure to Controlled in a post-processing operation. - -The control config syntax is as follows: -**hubblestack_data/hubblestack_nova_profiles/example_control/example.yaml** -```sh -control: - - CIS-2.1.4: This is the reason we control the check - - some_other_tag: - reason: This is the reason we control the check - - a_third_tag_with_no_reason - ``` -Note that providing a reason for the control is optional. Any of the three formats shown in the yaml list above will work. - -Once you have your compensating control config, just target the yaml to the hosts you want to control using your topfile. In this case, all the audits will still run, but if any of the controlled checks fail, they will be removed from Failure and added to Controlled, and will be treated as a Success for the purposes of compliance percentage. -To use the above control, you would add the following to your top.nova file: -```sh -nova: - '*': - - example_control.example -``` - -## Nebula -Nebula is Hubble’s Insight system, which ties into osquery, allowing you to query your infrastructure as if it were a database. This system can be used to take scheduled snapshots of your systems. - -Nebula leverages the osquery_nebula execution module which requires the osquery binary to be installed. More information about osquery can be found at `https://osquery.io`. - -### Usage - -Nebula queries have been designed to give detailed insight into system activity. The queries can be found in the following file. - -**hubblestack_data/hubblestack_nebula/hubblestack_nebula_queries.yaml** -```sh -fifteen_min: - - query_name: running_procs - query: SELECT p.name AS process, p.pid AS process_id, p.cmdline, p.cwd, p.on_disk, p.resident_size AS mem_used, p.parent, g.groupname, u.username AS user, p.path, h.md5, h.sha1, h.sha256 FROM processes AS p LEFT JOIN users AS u ON p.uid=u.uid LEFT JOIN groups AS g ON p.gid=g.gid LEFT JOIN hash AS h ON p.path=h.path; - - query_name: established_outbound - query: SELECT t.iso_8601 AS _time, pos.family, h.*, ltrim(pos.local_address, ':f') AS src, pos.local_port AS src_port, pos.remote_port AS dest_port, ltrim(remote_address, ':f') AS dest, name, p.path AS file_path, cmdline, pos.protocol, lp.protocol FROM process_open_sockets AS pos JOIN processes AS p ON p.pid=pos.pid LEFT JOIN time AS t LEFT JOIN (SELECT * FROM listening_ports) AS lp ON lp.port=pos.local_port AND lp.protocol=pos.protocol LEFT JOIN hash AS h ON h.path=p.path WHERE NOT remote_address='' AND NOT remote_address='::' AND NOT remote_address='0.0.0.0' AND NOT remote_address='127.0.0.1' AND port is NULL; - - query_name: listening_procs - query: SELECT t.iso_8601 AS _time, h.md5 AS md5, p.pid, name, ltrim(address, ':f') AS address, port, p.path AS file_path, cmdline, root, parent FROM listening_ports AS lp LEFT JOIN processes AS p ON lp.pid=p.pid LEFT JOIN time AS t LEFT JOIN hash AS h ON h.path=p.path WHERE NOT address='127.0.0.1'; - - query_name: suid_binaries - query: SELECT sb.*, t.iso_8601 AS _time FROM suid_bin AS sb JOIN time AS t; -hour: - - query_name: crontab - query: SELECT c.*,t.iso_8601 AS _time FROM crontab AS c JOIN time AS t; -day: - - query_name: rpm_packages - query: SELECT rpm.name, rpm.version, rpm.release, rpm.source AS package_source, rpm.size, rpm.sha1, rpm.arch, t.iso_8601 FROM rpm_packages AS rpm JOIN time AS t; -``` - -Nebula query data is best tracked in a central logging or similar system. However, if you would like to run the queries manually you can call the nebula execution module. -```sh -query_group : Group of queries to run -verbose : Defaults to False. If set to True, more information (such as the query which was run) will be included in the result. -``` -Examples: -```sh -hubble nebula.queries day -hubble nebula.queries hour [verbose=True] -hubble nebula.queries fifteen-min -``` -### Configuration -For Nebula module, configurations can be done via Nebula topfiles. Nebula topfile functionality is similar to Nova topfiles. - -**top.nebula topfile** - -```sh -nebula: - '*': - - hubblestack_nebula_queries - 'sample_team': - - sample_team_nebula_queries -``` -Nebula topfile, `nebula.top` by default has `hubblestack_nebula_queries.yaml` which consists queries as explained in the above usage section and if specific queries are required by teams then those queries can be added in a another yaml file and include it in `nebula.top` topfile. Place this new yaml file at the path `hubblestack_data/hubblestack_nebula` - -Examples for running `nebula.top`: -```sh -hubble nebula.top hour -hubble nebula.top foo/bar/top.nova hour -hubble nebula.top fifteen_min verbose=True -``` -## Pulsar - -Pulsar is designed to monitor for file system events, acting as a real-time File Integrity Monitoring (FIM) agent. Pulsar is composed of a custom Salt beacon that watches for these events and hooks into the returner system for alerting and reporting. In other words, you can receive real-time alerts for unscheduled file system modifications anywhere you want to receive them. We’ve designed Pulsar to be lightweight and does not affect the system performance. It simply watches for events and directly sends them to one of the Pulsar returner destinations. - -### Usage -Once Pulsar is configured there isn’t anything you need to do to interact with it. It simply runs quietly in the background and sends you alerts. - -### Configuration -The Pulsar configuration can be found at `hubblestack_pulsar_config.yaml` file. Every environment will have different needs and requirements, and we understand that, so we’ve designed Pulsar to be flexible. - -**hubblestack_pulsar_config.yaml** -```sh -/etc: { recurse: True, auto_add: True } -/bin: { recurse: True, auto_add: True } -/sbin: { recurse: True, auto_add: True } -/boot: { recurse: True, auto_add: True } -/usr/bin: { recurse: True, auto_add: True } -/usr/sbin: { recurse: True, auto_add: True } -/usr/local/bin: { recurse: True, auto_add: True } -/usr/local/sbin: { recurse: True, auto_add: True } -return: slack_pulsar -checksum: sha256 -stats: True -batch: False -``` - -Pulsar runs on schedule which can be found at `/etc/hubble/hubble` - -**/etc/hubble/hubble** -```sh -schedule: - pulsar: - function: pulsar.process - seconds: 1 - returner: splunk_pulsar_return - run_on_start: True - pulsar_canary: - function: pulsar.canary - seconds: 86400 - job1: - function: hubble.audit - seconds: 60 - splay: 30 - args: - - cis.centos-7-level-1-scored-v2-1-0 - kwargs: - verbose: True - returner: splunk_nova_return - run_on_start: True -``` - -In order to receive Pulsar notifications you’ll need to install the custom returners found in the Quasar repository. - -Example of using the Slack Pulsar returner to receive FIM notifications: -```sh -slack_pulsar: - as_user: true - username: calculon - channel: channel - api_key: xoxb-xxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx -``` -#### Excluding Paths -There may be certain paths that you want to exclude from this real-time FIM tool. This can be done using the exclude: keyword beneath any defined path in `hubblestack_pulsar_config.yaml` file. - -/var: - recurse: True - auto_add: True - exclude: - - /var/log - - /var/spool - - /var/cache - - /var/lock - -For Pulsar module, configurations can be done via Pulsar topfiles when teams needs to add specific configurations or exclusions as discussed above. - -#### Pulsar topfile `top.pulsar` - -For Pulsar module, configurations can be done via Pulsar topfiles - -```sh -pulsar: - '*': - - hubblestack_pulsar_config - 'sample_team': - - sample_team_hubblestack_pulsar_config -``` -Pulsar topfile by default has 'hubblestack_pulsar_config' which consists of default configurations and if specific configurations are required by teams then those can be added in another yaml file and include it in 'pulsar.top' topfile. Place this new yaml file at the path `hubblestack/hubblestack_data/hubblestack_pulsar` - -Examples for running pulsar.top: -```sh -hubble pulsar.top -hubble pulsar.top verbose=True -``` diff --git a/README.rst b/README.rst new file mode 100644 index 000000000..b3c544f46 --- /dev/null +++ b/README.rst @@ -0,0 +1,8 @@ +Welcome to HubbleStack! +======================= + +You can find the docs `here `_ + +You can file an issue `here `_ diff --git a/doc/contents/auditing_nova.rst b/doc/contents/auditing_nova.rst index 495f32eaf..4d291c576 100644 --- a/doc/contents/auditing_nova.rst +++ b/doc/contents/auditing_nova.rst @@ -4,6 +4,11 @@ Auditing (Nova) Hubble supports success/fail auditing via a number of included modules. The codename for the audit piece of hubble is "Nova". +Module Documentation +-------------------- + +:doc:`modules/nova` + Usage ----- diff --git a/doc/contents/fim_pulsar.rst b/doc/contents/fim_pulsar.rst new file mode 100644 index 000000000..d2220eb33 --- /dev/null +++ b/doc/contents/fim_pulsar.rst @@ -0,0 +1,112 @@ +File Integrity Monitoring/FIM (Linux) (Pulsar) +============================================== + +Pulsar is designed to monitor for file system events, acting as a real-time +File Integrity Monitoring (FIM) agent. Pulsar uses python-inotify to watch for +these events and report them to your destination of choice. + +Module Documentation +-------------------- + +:doc:`modules/pulsar` + +Usage +----- + +Once Pulsar is configured there isn’t anything you need to do to interact with +it. It simply runs quietly in the background and sends you alerts. + +.. note:: + + Running pulsar outside of hubble's scheduler will never return results. + This is because the first time you run pulsar it will set up the watches in + inotify, but no events will have been generated. Only subsequent runs under + the same process can receive events. + +Configuration +------------- + +The list of files and directories that pulsar watches is defined in +salt://hubblestack_pulsar/hubblestack_pulsar_config.yaml:: + + /lib: { recurse: True, auto_add: True } + /bin: { recurse: True, auto_add: True } + /sbin: { recurse: True, auto_add: True } + /boot: { recurse: True, auto_add: True } + /lib64: { recurse: True, auto_add: True } + /usr/lib: { recurse: True, auto_add: True } + /usr/bin: { recurse: True, auto_add: True } + /usr/sbin: { recurse: True, auto_add: True } + /usr/lib64: { recurse: True, auto_add: True } + /usr/libexec: { recurse: True, auto_add: True } + /usr/local/etc: { recurse: True, auto_add: True } + /usr/local/bin: { recurse: True, auto_add: True } + /usr/local/lib: { recurse: True, auto_add: True } + /usr/local/sbin: { recurse: True, auto_add: True } + /usr/local/libexec: { recurse: True, auto_add: True } + /opt/bin: { recurse: True, auto_add: True } + /opt/osquery: { recurse: True, auto_add: True } + /opt/hubble: { recurse: True, auto_add: True } + /etc: + exclude: + - /etc/passwd.lock + - /etc/shadow.lock + - /etc/gshadow.lock + - /etc/group.lock + - /etc/passwd+ + - /etc/passwd- + - /etc/shadow+ + - /etc/shadow- + - /etc/group+ + - /etc/group- + - /etc/gshadow+ + - /etc/gshadow- + - /etc/cas/timestamp + - /etc/resolv.conf.tmp + - /etc/pki/nssdb/key4.db-journal + - /etc/pki/nssdb/cert9.db-journal + - /etc/salt/gpgkeys/random_seed + - /etc/blkid/blkid.tab.old + - \/etc\/blkid\/blkid\.tab\-\w{6}$: + regex: True + - \/etc\/passwd\.\d*$: + regex: True + - \/etc\/group\.\d*$: + regex: True + - \/etc\/shadow\.\d*$: + regex: True + - \/etc\/gshadow\.\d*$: + regex: True + recurse: True + auto_add: True + return: splunk_pulsar_return + checksum: sha256 + stats: True + batch: True + +Note some of the available options: you can recurse through directories, +auto_add new files and directories as they are created, or exclude based on +glob or regex patterns. + +topfiles +^^^^^^^^ + +Pulsar supports organizing query groups across files, and combining/targeting +them via a ``top.pulsar`` file (similar to topfiles in SaltStack):: + + pulsar: + '*': + - hubblestack_pulsar_config + +Each entry under ``pulsar`` is a SaltStack style `compound match`_ that +describes which hosts should receive the list of queries. All queries are +merged, and conflicts go to the last-defined file. + +The files referenced are relative to ``salt://hubblestack_pulsar/`` and +leave off the ``.yaml`` extension. + +You can also specify an alternate ``top.pulsar`` file. + +For more details, see the module documentation: :doc:`modules/pulsar` + +.. _compound match: https://docs.saltstack.com/en/latest/topics/targeting/compound.html diff --git a/doc/contents/fim_win_pulsar.rst b/doc/contents/fim_win_pulsar.rst new file mode 100644 index 000000000..f13eb16b5 --- /dev/null +++ b/doc/contents/fim_win_pulsar.rst @@ -0,0 +1,29 @@ +File Integrity Monitoring/FIM (Windows) (Pulsar) +================================================ + +Pulsar for Windows is designed to monitor for file system events, acting as a +real-time File Integrity Monitoring (FIM) agent. On Windows systems, pulsar +uses ntfs journaling watch for these events and report them to your destination +of choice. + +Module Documentation +-------------------- + +:doc:`modules/win_pulsar` + +Usage +----- + +Once Pulsar is configured there isn’t anything you need to do to interact with +it. It simply runs quietly in the background and sends you alerts. + +.. note:: + + Running pulsar outside of hubble's scheduler will never return results. + This is because the first time you run pulsar it will set up the watches in + inotify, but no events will have been generated. Only subsequent runs under + the same process can receive events. + +.. todo:: + + UM WHERE ARE THE DOCS diff --git a/doc/contents/insight_nebula.rst b/doc/contents/insight_nebula.rst new file mode 100644 index 000000000..d1ccbd63d --- /dev/null +++ b/doc/contents/insight_nebula.rst @@ -0,0 +1,74 @@ +Insights and Data Gathering (Nebula) +==================================== + +Hubble can gather incredible amounts of raw data from your hosts for later +analysis. The codename for the insights piece of hubble is Nebula. It primarily +uses `osquery `_ which allows you to query your system as +if it were a database. + +Module Documentation +-------------------- + +:doc:`modules/nebula_osquery` + +Usage +----- + +Nebula queries are formatted into query groups which allow you to schedule +sets of queries to run at different cadences. The names of these groups are +arbitrary, but the queries provided in `hubblestack_data`_ are grouped by +timing:: + + fifteen_min: + running_procs: + query: SELECT t.unix_time AS query_time, p.name AS process, p.pid AS process_id, p.pgroup AS process_group, p.cmdline, p.cwd, p.on_disk, p.resident_size AS mem_used, p.user_time, p.system_time, (SELECT strftime('%s','now')-ut.total_seconds+p.start_time FROM uptime AS ut) AS process_start_time, p.parent, pp.name AS parent_name, g.groupname AS 'group', g.gid AS group_id, u.username AS user, u.uid AS user_id, eu.username AS effective_username, eg.groupname AS effective_groupname, p.path, h.md5 AS md5, h.sha1 AS sha1, h.sha256 AS sha256, '__JSONIFY__'||(SELECT json_group_array(json_object('fd',pof.fd, 'path',pof.path)) FROM process_open_files AS pof WHERE pof.pid=p.pid GROUP BY pof.pid) AS open_files, '__JSONIFY__'||(SELECT json_group_array(json_object('variable_name',pe.key, 'value',pe.value)) FROM process_envs AS pe WHERE pe.pid=p.pid GROUP BY pe.pid) AS environment FROM processes AS p LEFT JOIN processes AS pp ON p.parent=pp.pid LEFT JOIN users AS u ON p.uid=u.uid LEFT JOIN users AS eu ON p.euid=eu.uid LEFT JOIN groups AS g ON p.gid=g.gid LEFT JOIN groups AS eg ON p.gid=eg.gid LEFT JOIN hash AS h ON p.path=h.path LEFT JOIN time AS t WHERE p.parent IS NOT 2 AND (process NOTNULL OR p.parent NOTNULL); + established_outbound: + query: SELECT t.unix_time AS query_time, CASE pos.family WHEN 2 THEN 'ipv4' WHEN 10 THEN 'ipv6' ELSE pos.family END AS family, h.md5 AS md5, h.sha1 AS sha1, h.sha256 AS sha256, h.directory AS directory, ltrim(pos.local_address, ':f') AS src_connection_ip, pos.local_port AS src_connection_port, pos.remote_port AS dest_connection_port, ltrim(pos.remote_address, ':f') AS dest_connection_ip, p.name AS name, p.pid AS pid, p.parent AS parent_pid, pp.name AS parent_process, p.path AS file_path, f.size AS file_size, p.cmdline AS cmdline, u.uid AS uid, u.username AS username, CASE pos.protocol WHEN 6 THEN 'tcp' WHEN 17 THEN 'udp' ELSE pos.protocol END AS transport FROM process_open_sockets AS pos JOIN processes AS p ON p.pid=pos.pid LEFT JOIN processes AS pp ON p.parent=pp.pid LEFT JOIN users AS u ON p.uid=u.uid LEFT JOIN time AS t LEFT JOIN hash AS h ON h.path=p.path LEFT JOIN file AS f ON f.path=p.path WHERE NOT pos.remote_address='' AND NOT pos.remote_address='::' AND NOT pos.remote_address='0.0.0.0' AND NOT pos.remote_address='127.0.0.1' AND (pos.local_port,pos.protocol) NOT IN (SELECT lp.port, lp.protocol FROM listening_ports AS lp); + hour: + crontab: + query: SELECT t.unix_time AS query_time, c.event, c.minute, c.hour, c.day_of_month, c.month, c.day_of_week, c.command, c.path AS cron_file FROM crontab AS c JOIN time AS t; + login_history: + query: SELECT t.unix_time AS query_time, l.username AS user, l.tty, l.pid, l.type AS utmp_type, CASE l.type WHEN 1 THEN 'RUN_LVL' WHEN 2 THEN 'BOOT_TIME' WHEN 3 THEN 'NEW_TIME' WHEN 4 THEN 'OLD_TIME' WHEN 5 THEN 'INIT_PROCESS' WHEN 6 THEN 'LOGIN_PROCESS' WHEN 7 THEN 'USER_PROCESS' WHEN 8 THEN 'DEAD_PROCESS' ELSE l.type END AS utmp_type_name, l.host AS src, l.time FROM last AS l LEFT JOIN time AS t WHERE l.time > strftime('%s','now') - 3600; + docker_running_containers: + query: SELECT t.unix_time AS query_time, dc.id AS container_id, dc.name AS container_name, dc.image AS image_name, di.created AS image_created_time, di.size_bytes AS image_size, di.tags AS image_tags, dc.image_id AS image_id, dc.command AS container_command, dc.created AS container_start_time, dc.state AS container_state, dc.status AS status, '__JSONIFY__'||(SELECT json_group_array(json_object('key',dcl.key, 'value',dcl.value)) FROM docker_container_labels AS dcl WHERE dcl.id=dc.id GROUP BY dcl.id) AS container_labels, '__JSONIFY__'||(SELECT json_group_array(json_object('mount_type',dcm.type, 'mount_name',dcm.name, 'mount_host_path',dcm.source, 'mount_container_path',dcm.destination, 'mount_driver',dcm.driver, 'mount_mode',dcm.mode, 'mount_rw',dcm.rw, 'mount_progpagation',dcm.propagation)) FROM docker_container_mounts AS dcm WHERE dcm.id=dc.id GROUP BY dcm.id) AS container_mounts, '__JSONIFY__'||(SELECT json_group_array(json_object('port_type',dcport.type, 'port',dcport.port, 'host_ip',dcport.host_ip, 'host_port',dcport.host_port)) FROM docker_container_ports AS dcport WHERE dcport.id=dc.id GROUP BY dcport.id) AS container_ports, '__JSONIFY__'||(SELECT json_group_array(json_object('network_name',dcnet.name, 'network_id',dcnet.network_id, 'endpoint_id',dcnet.endpoint_id, 'gateway',dcnet.gateway, 'container_ip',dcnet.ip_address, 'container_ip_prefix_len',dcnet.ip_prefix_len, 'ipv6_gateway',dcnet.ipv6_gateway, 'container_ipv6_address',dcnet.ipv6_address, 'container_ipv6_prefix_len',dcnet.ipv6_prefix_len, 'container_mac_address',dcnet.mac_address)) FROM docker_container_networks AS dcnet WHERE dcnet.id=dc.id GROUP BY dcnet.id) AS container_networks FROM docker_containers AS dc JOIN docker_images AS di ON di.id=dc.image_id LEFT JOIN time AS t; + day: + rpm_packages: + query: SELECT t.unix_time AS query_time, rpm.name, rpm.version, rpm.release, rpm.source AS package_source, rpm.size, rpm.sha1, rpm.arch FROM rpm_packages AS rpm JOIN time AS t; + os_info: + query: SELECT t.unix_time AS query_time, os.* FROM os_version AS os LEFT JOIN time AS t; + interface_addresses: + query: SELECT t.unix_time AS query_time, ia.interface, ia.address, id.mac FROM interface_addresses AS ia JOIN interface_details AS id ON ia.interface=id.interface LEFT JOIN time AS t WHERE NOT ia.interface='lo'; + +Nebula query data is very verbose, and is really meant to be sent to a central +aggregation location such as splunk or logstash for further processing. +However, if you would like to run the queries manually you can call the :doc:`nebula +execution module `:: + + hubble nebula.queries day + hubble nebula.queries hour verbose=True + +topfiles +-------- + +Nebula supports organizing query groups across files, and combining/targeting +them via a ``top.nebula`` file (similar to topfiles in SaltStack):: + + nebula: + - '*': + - hubblestack_nebula_queries + - 'G@splunk_index:some_team': + - some_team + +Each entry under ``nebula`` is a SaltStack style `compound match`_ that +describes which hosts should receive the list of queries. All queries are +merged, and conflicts go to the last-defined file. + +The files referenced are relative to ``salt://hubblestack_nebula_v2/`` and +leave off the ``.yaml`` extension. + +You can also specify an alternate ``top.nebula`` file. + +For more details, see the module documentation: :doc:`modules/nebula_osquery` + +.. _hubblestack_data: https://github.com/hubblestack/hubblestack_data/blob/develop/hubblestack_nebula_v2/hubblestack_nebula_queries.yaml +.. _compound match: https://docs.saltstack.com/en/latest/topics/targeting/compound.html diff --git a/doc/index.rst b/doc/index.rst index 73d74a084..2685fa839 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -16,6 +16,9 @@ Table of Contents: contents/gettingstarted contents/auditing_nova + contents/insight_nebula + contents/fim_pulsar + contents/fim_win_pulsar contents/module_documentation Indices and Tables: From 66d4bdb0f0323786ad595a94ec3c59cb97b19186 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 15:56:57 -0600 Subject: [PATCH 20/23] Add salt-ssh to docs requirements --- doc/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/requirements.txt b/doc/requirements.txt index 04c857701..286758fcb 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,3 +1,4 @@ sphinx sphinx-autobuild sphinx_rtd_theme +salt-ssh From 200dc8bc2075da5753b2424f1597d806a4139d41 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 7 Aug 2018 16:03:48 -0600 Subject: [PATCH 21/23] Fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b3c544f46..99559bc03 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,6 @@ Welcome to HubbleStack! You can find the docs `here `_ -You can file an issue `here `_ Follow us on `Twitter! `_ From 406bb730c26405720091d2dd4ec8524953fea6c4 Mon Sep 17 00:00:00 2001 From: Chad Mills Date: Wed, 15 Aug 2018 23:24:05 -0600 Subject: [PATCH 22/23] This dockerfile will create a docker container that will build hubble and drop it in a folder of your choosing on your host machine --- pkg/windows/dockerfile | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 pkg/windows/dockerfile diff --git a/pkg/windows/dockerfile b/pkg/windows/dockerfile new file mode 100644 index 000000000..02d95c618 --- /dev/null +++ b/pkg/windows/dockerfile @@ -0,0 +1,85 @@ +# This Dockerfile aims to make building Hubble v2 packages easier. +# If you don't have docker installed on your server/workstation please run setup_docker_env.ps1 +# To build an image: 1. copy pkg/windows/pyinstaller-requirements.txt & to directory with this Dockerfile +# 2. docker build -t . +# The resulting image is ready to run the pyinstaller on container start and drop hubble.exe +# in a local directory. Mount /data volume into a directory on the host to access the package. +# To run the container: docker run -it --rm -v :c:\data +#build docker image from windowscore +FROM microsoft/windowsservercore +#Needed to just work +ENV PYTHONIOENCODING='UTF-8' +ENV CHOCO_URL=https://chocolatey.org/install.ps1 +#All the variables used for salt +ENV SALT_SRC_PATH='C:/temp/salt/' +ENV SALT_GIT_URL=https://github.com/saltstack/salt +ENV SALT_CHECKOUT=v2018.3.0 +#All the variables used for hubble +ENV HUBBLE_CHECKOUT=v2.2.4-2 +ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ENV HUBBLE_SRC_PATH='C:/temp/hubble/' +ENV _HOOK_DIR='./pkg/' +ENV NSIS_LOC='C:/Program Files (x86)/NSIS' +#Create location for build environment and set as working dir +RUN powershell.exe -Command New-Item c:/temp -ItemType Directory; \ + New-Item C:/data -ItemType Directory; +WORKDIR C:/temp +VOLUME C:/data +#Copy local files to working directory +COPY pyinstaller-requirements.txt c:/temp/ +COPY hubble.conf C:/temp/ +#install Chocolatey, then git and git.portable, and osquery +RUN powershell.exe -Command Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString("$env:CHOCO_URL")) +RUN powershell.exe -Command choco install git git.portable osquery nssm -y; + +#RUN powershell.exe $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") +#Git clone salt and run packages +RUN powershell.exe -Command git clone "$env:SALT_GIT_URL"; \ + Push-Location salt/pkg/windows; \ + git checkout "$env:SALT_CHECKOUT"; \ + C:/temp/salt/pkg/windows/build_env_2.ps1 -Silent; +#more salt installs +RUN powershell.exe -Command Push-Location salt; \ + test-path ./setup.py; \ + python ./setup.py --quiet install --force; \ + pop-location; +#Clone Hubble +RUN powershell.exe -Command git clone "$env:HUBBLE_GIT_URL"; \ + Push-Location hubble; \ + git checkout "$env:HUBBLE_CHECKOUT"; \ + pop-location; + +#Use pip to install hubble dependencies +RUN powershell.exe -Command pip install -r pyinstaller-requirements.txt; + +#Move portable git to a new location +RUN powershell.exe -Command New-Item C:/temp/hubble/PortableGit -ItemType Directory; \ + Copy-Item -Path C:/tools/git -Destination C:/temp/hubble/PortableGit -Recurse; + +# Modify gitfs fix for incorrect path variables until fix has been upstreamed +RUN powershell.exe -Command If (!(Test-Path C:/Python27/Lib/site-packages/salt)) {Copy-Item C:/temp/salt/salt -Destination C:/Python27/Lib/site-packages/ -Recurse -Force}; \ + $gitfsFile = Get-Content C:\Python27\Lib\site-packages\salt\utils\gitfs.py; \ + $gitfsFile = $gitfsFile -replace 'files.add\\(add_mountpoint\\(relpath\\(repo_path\\)\\)\\)','files.add("/".join(repo_path.partition(".:\\")[2].split(os.sep)))'; \ + Set-Content -Path C:\Python27\Lib\site-packages\salt\utils\gitfs.py -Value $gitfsFile -Force +#Get vcredist prereq for hubble +RUN powershell.exe -Command \ + $ProgressPreference = 'SilentlyContinue'; \ + Invoke-WebRequest -Uri 'http://repo.saltstack.com/windows/dependencies/64/vcredist_x64_2008_mfc.exe' -OutFile "C:/temp/hubble/pkg/windows/vcredist.exe" +#Create pyionstaller spec and edit it to work with windows +CMD powershell.exe -Command Push-Location C:/temp/hubble; \ + pyi-makespec --additional-hooks-dir=$env:_HOOK_DIR ./hubble.py; \ + $specFile = Get-Content 'C:/temp/hubble/hubble.spec'; \ + $specFile = $specFile -replace 'a.binaries','a.binaries + [(''libeay32.dll'', ''C:\Python27\libeay32.dll'', ''BINARY'')]'; \ + Set-Content -Path ./hubble.spec -Value $specFile -Force; \ + pyinstaller ./hubble.spec; \ + Pop-Location; \ +#Move the hubble.conf, PortableGit, nssm, and osquery to the corerect location + New-Item './hubble/dist/hubble/etc/hubble' -ItemType Directory; \ + Move-Item hubble.conf -Destination ./hubble/dist/hubble/etc/hubble/; \ + Move-Item 'hubble/PortableGit' -Destination './hubble/dist/hubble/' -Force; \ + Move-Item 'C:/ProgramData/chocolatey/lib/NSSM/tools/nssm.exe' -Destination './hubble/dist/hubble/' -Force; \ + Move-Item 'C:/ProgramData/osquery/osqueryi.exe' -Destination './hubble/pkg/' -Force; \ +#Build the installer + Push-Location 'C:/Program Files (x86)/NSIS'; \ + ./makensis.exe /DHubbleVersion="$env:HUBBLE_CHECKOUT" 'C:/temp/hubble/pkg/windows/hubble-Setup.nsi'; \ + Copy-Item C:/temp/hubble/pkg/windows/Hubble*exe -Destination C:/data/ \ No newline at end of file From 3818d423bcc14a0ed5bdbe2adac86db3fe02bf09 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Tue, 21 Aug 2018 15:54:36 -0600 Subject: [PATCH 23/23] Rev to version 2.4.2 --- doc/conf.py | 4 ++-- hubblestack/__init__.py | 2 +- pkg/amazonlinux2016.09/Dockerfile | 4 ++-- pkg/centos6/Dockerfile | 4 ++-- pkg/centos7/Dockerfile | 4 ++-- pkg/coreos/Dockerfile | 4 ++-- pkg/debian7/Dockerfile | 4 ++-- pkg/debian8/Dockerfile | 4 ++-- pkg/debian9/Dockerfile | 4 ++-- pkg/dev/amazonlinux2016.09/Dockerfile | 2 +- pkg/dev/centos6/Dockerfile | 2 +- pkg/dev/centos7/Dockerfile | 2 +- pkg/dev/coreos/Dockerfile | 2 +- pkg/dev/debian7/Dockerfile | 2 +- pkg/dev/debian8/Dockerfile | 2 +- pkg/dev/debian9/Dockerfile | 2 +- 16 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 581a31ab6..803b87d3a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = u'2.4.1' +version = u'2.4.2' # The full version, including alpha/beta/rc tags. -release = u'2.4.1-1' +release = u'2.4.2-1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/hubblestack/__init__.py b/hubblestack/__init__.py index 5cd7abf83..cb9dc8a91 100644 --- a/hubblestack/__init__.py +++ b/hubblestack/__init__.py @@ -1 +1 @@ -__version__ = '2.4.1' +__version__ = '2.4.2' diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index ba29ec09e..3050dcf80 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -125,8 +125,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile index c7feb0c74..7ed82dcf0 100644 --- a/pkg/centos6/Dockerfile +++ b/pkg/centos6/Dockerfile @@ -127,8 +127,8 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index 97ea87594..218232536 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -124,8 +124,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index 2de28ef71..370956f9a 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -136,9 +136,9 @@ RUN pip -v install -r pyinstaller-requirements.txt #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/debian7/Dockerfile b/pkg/debian7/Dockerfile index 010e8cb94..65024e348 100644 --- a/pkg/debian7/Dockerfile +++ b/pkg/debian7/Dockerfile @@ -163,9 +163,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 8cdf6354e..30c17ee45 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -145,9 +145,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index a8a3f658a..7440593a5 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -140,9 +140,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ENV HUBBLE_CHECKOUT=v2.4.1 +ENV HUBBLE_CHECKOUT=v2.4.2 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1 +ENV HUBBLE_VERSION=2.4.2 ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/amazonlinux2016.09/Dockerfile b/pkg/dev/amazonlinux2016.09/Dockerfile index 537ff1f3a..c8c74cf1e 100644 --- a/pkg/dev/amazonlinux2016.09/Dockerfile +++ b/pkg/dev/amazonlinux2016.09/Dockerfile @@ -128,7 +128,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \ #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/centos6/Dockerfile b/pkg/dev/centos6/Dockerfile index d27f8b506..d0a045c77 100644 --- a/pkg/dev/centos6/Dockerfile +++ b/pkg/dev/centos6/Dockerfile @@ -130,7 +130,7 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \ #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/centos7/Dockerfile b/pkg/dev/centos7/Dockerfile index 8d0a0bde3..00d7e07c5 100644 --- a/pkg/dev/centos7/Dockerfile +++ b/pkg/dev/centos7/Dockerfile @@ -127,7 +127,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \ #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/coreos/Dockerfile b/pkg/dev/coreos/Dockerfile index d95875a0d..aad218e91 100644 --- a/pkg/dev/coreos/Dockerfile +++ b/pkg/dev/coreos/Dockerfile @@ -139,7 +139,7 @@ RUN pip -v install -r pyinstaller-requirements.txt #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/debian7/Dockerfile b/pkg/dev/debian7/Dockerfile index 6822b0005..f9eb19d96 100644 --- a/pkg/dev/debian7/Dockerfile +++ b/pkg/dev/debian7/Dockerfile @@ -166,7 +166,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/debian8/Dockerfile b/pkg/dev/debian8/Dockerfile index 8d879d54e..baa8a4fcd 100644 --- a/pkg/dev/debian8/Dockerfile +++ b/pkg/dev/debian8/Dockerfile @@ -148,7 +148,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" diff --git a/pkg/dev/debian9/Dockerfile b/pkg/dev/debian9/Dockerfile index 08165f33f..b41a2202c 100644 --- a/pkg/dev/debian9/Dockerfile +++ b/pkg/dev/debian9/Dockerfile @@ -143,7 +143,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \ #use the following variables to choose the version of hubble ARG HUBBLE_CHECKOUT=develop ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git -ENV HUBBLE_VERSION=2.4.1_develop +ENV HUBBLE_VERSION=2.4.2_develop ENV HUBBLE_ITERATION=1 ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/"