From 77c15cc2ba8fa8a23bd4bbb7115beec93c34205b Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 29 Aug 2016 13:39:07 -0600 Subject: [PATCH] Split out most of the beacon config to salt fileserver (#17) * Split out pulsar to fileserver config * Remove outdated validate function * Add the post-compile log * Log in trace, not garbage * Update readme and example pillar * Remove old data and add win_pillar.example --- README.rst | 62 ++++---- _beacons/pulsar.py | 130 ++++++---------- hubblestack_pulsar_config.yaml | 48 ++++++ pillar.example | 50 +----- pulsar/_modules/checksum.py | 144 ------------------ pulsar/inotify/etc/salt/master.d/reactor.conf | 3 - pulsar/inotify/pillar/beacons.sls | 31 ---- pulsar/inotify/pillar/hubblestack.sls | 5 - pulsar/inotify/pillar/top.sls | 5 - pulsar/inotify/reactor/slack.sls | 32 ---- pulsar/win_notify/pillar/top.sls | 3 - .../pillar/beacons.sls => win_pillar.example | 0 12 files changed, 134 insertions(+), 379 deletions(-) create mode 100644 hubblestack_pulsar_config.yaml delete mode 100644 pulsar/_modules/checksum.py delete mode 100644 pulsar/inotify/etc/salt/master.d/reactor.conf delete mode 100644 pulsar/inotify/pillar/beacons.sls delete mode 100644 pulsar/inotify/pillar/hubblestack.sls delete mode 100644 pulsar/inotify/pillar/top.sls delete mode 100644 pulsar/inotify/reactor/slack.sls delete mode 100644 pulsar/win_notify/pillar/top.sls rename pulsar/win_notify/pillar/beacons.sls => win_pillar.example (100%) diff --git a/README.rst b/README.rst index fd5bcfd..ecb89d8 100644 --- a/README.rst +++ b/README.rst @@ -100,8 +100,8 @@ You should now be able to sync the new modules to your minion(s) using the salt \* saltutil.sync_beacons -Copy the ``hubblestack_pulsar.sls.orig`` into your Salt pillar, dropping the -``.orig`` extension and target it to selected minions. +Copy the ``pillar.example`` into your Salt pillar, renaming is as desired +(perhaps ``hubblestack_pulsar.sls``) and target it to selected minions. .. code-block:: shell @@ -131,6 +131,7 @@ the minions. cd hubblestack-pulsar.git mkdir -p /srv/salt/_beacons/ cp _beacons/pulsar.py /srv/salt/_beacons/ + cp hubblestack_pulsar_config.yaml /srv/salt/ cp pillar.example /srv/pillar/hubblestack_pulsar.sls salt \* saltutil.sync_beacons @@ -159,27 +160,34 @@ with it. It simply runs quietly in the background and sends you alerts. Configuration ============= -The default Pulsar configuration (found in ```` -is meant to act as a template. Every environment will have different needs and -requirements, and we understand that, so we've designed Pulsar to be flexible. +The default Pulsar configuration (found in ````) +is meant to act as a template. It works in tandem with the +```` file. Every environment will have +different needs and requirements, and we understand that, so we've designed +Pulsar to be flexible. .. code-block:: yaml + # pillar.example beacons: pulsar: - /etc: { recurse: True, auto_add: True } - /bin: { recurse: True, auto_add: True } - /sbin: { recurse: True, auto_add: True } - /boot: { recurse: True, auto_add: True } - /usr/bin: { recurse: True, auto_add: True } - /usr/sbin: { recurse: True, auto_add: True } - /usr/local/bin: { recurse: True, auto_add: True } - /usr/local/sbin: { recurse: True, auto_add: True } - - return: slack_pulsar - checksum: sha256 - stats: True - batch: False + paths: + - salt://hubblestack_pulsar_config.yaml + + + # hubblestack_pulsar_config.yaml + /etc: { recurse: True, auto_add: True } + /bin: { recurse: True, auto_add: True } + /sbin: { recurse: True, auto_add: True } + /boot: { recurse: True, auto_add: True } + /usr/bin: { recurse: True, auto_add: True } + /usr/sbin: { recurse: True, auto_add: True } + /usr/local/bin: { recurse: True, auto_add: True } + /usr/local/sbin: { recurse: True, auto_add: True } + return: slack_pulsar + checksum: sha256 + stats: True + batch: False In order to receive Pulsar notifications you'll need to install the custom returners found in the Quasar_ repository. @@ -207,16 +215,14 @@ defined path. .. code-block:: yaml - beacons: - pulsar: - /var: - recurse: True - auto_add: True - exclude: - - /var/log - - /var/spool - - /var/cache - - /var/lock + /var: + recurse: True + auto_add: True + exclude: + - /var/log + - /var/spool + - /var/cache + - /var/lock .. _pulsar_under_the_hood: diff --git a/_beacons/pulsar.py b/_beacons/pulsar.py index 5bd3627..a0cc479 100644 --- a/_beacons/pulsar.py +++ b/_beacons/pulsar.py @@ -17,6 +17,7 @@ import fnmatch import os import re +import yaml # Import salt libs import salt.ext.six @@ -73,96 +74,43 @@ def _get_notifier(): return __context__['pulsar.notifier'] -def validate(config): - ''' - Validate the beacon configuration - ''' - - VALID_MASK = [ - 'access', - 'attrib', - 'close_nowrite', - 'close_write', - 'create', - 'delete', - 'delete_self', - 'excl_unlink', - 'ignored', - 'modify', - 'moved_from', - 'moved_to', - 'move_self', - 'oneshot', - 'onlydir', - 'open', - 'unmount' - ] - - # Configuration for pulsar beacon should be a dict of dicts - log.debug('config {0}'.format(config)) - if not isinstance(config, dict): - return False, 'Configuration for pulsar beacon must be a dictionary.' - else: - for config_item in config: - if not isinstance(config[config_item], dict): - return False, ('Configuration for pulsar beacon must ' - 'be a dictionary of dictionaries.') - else: - if not any(j in ['mask', 'recurse', 'auto_add'] for j in config[config_item]): - return False, ('Configuration for pulsar beacon must ' - 'contain mask, recurse or auto_add items.') - - if 'auto_add' in config[config_item]: - if not isinstance(config[config_item]['auto_add'], bool): - return False, ('Configuration for pulsar beacon ' - 'auto_add must be boolean.') - - if 'recurse' in config[config_item]: - if not isinstance(config[config_item]['recurse'], bool): - return False, ('Configuration for pulsar beacon ' - 'recurse must be boolean.') - - if 'mask' in config[config_item]: - if not isinstance(config[config_item]['mask'], list): - return False, ('Configuration for pulsar beacon ' - 'mask must be list.') - for mask in config[config_item]['mask']: - if mask not in VALID_MASK: - return False, ('Configuration for pulsar beacon ' - 'invalid mask option {0}.'.format(mask)) - return True, 'Valid beacon configuration' - - def beacon(config): ''' Watch the configured files - Example Config + Example pillar config .. code-block:: yaml beacons: pulsar: - /path/to/file/or/dir: - mask: - - open - - create - - close_write - recurse: True - auto_add: True - exclude: - - /path/to/file/or/dir/exclude1 - - /path/to/file/or/dir/exclude2 - - /path/to/file/or/dir/regex[\d]*$: - regex: True - return: - splunk: - batch: True - slack: - batch: False # overrides the global setting - checksum: sha256 - stats: True + paths: + - salt://hubblestack_pulsar_config.yaml + + Example yaml config on fileserver (targeted by pillar) + + .. code-block:: yaml + + /path/to/file/or/dir: + mask: + - open + - create + - close_write + recurse: True + auto_add: True + exclude: + - /path/to/file/or/dir/exclude1 + - /path/to/file/or/dir/exclude2 + - /path/to/file/or/dir/regex[\d]*$: + regex: True + return: + splunk: batch: True + slack: + batch: False # overrides the global setting + checksum: sha256 + stats: True + batch: True Note that if `batch: True`, the configured returner must support receiving a list of events, rather than single one-off events. @@ -201,10 +149,32 @@ def beacon(config): If pillar/grains/minion config key `hubblestack:pulsar:maintenance` is set to True, then changes will be discarded. ''' + log.debug('Pulsar beacon called.') + log.trace('Pulsar beacon config from pillar:\n{0}'.format(config)) ret = [] notifier = _get_notifier() wm = notifier._watch_manager + # Get config(s) from salt fileserver + new_config = config + if isinstance(config.get('paths'), list): + for path in config['paths']: + cpath = __salt__['cp.cache_file'](path) + if os.path.isfile(cpath): + with open(cpath, 'r') as f: + new_config = salt.utils.dictupdate.update(new_config, + yaml.safe_load(f), + recursive_update=True, + merge_lists=True) + else: + log.error('Path {0} does not exist or is not a file'.format(cpath)) + else: + log.error('Pulsar beacon \'paths\' data improperly formatted. Should be list of salt:// paths') + + config = new_config + + log.trace('Pulsar beacon config (compiled from config list):\n{0}'.format(config)) + # Read in existing events if notifier.check_events(1): notifier.read_events() diff --git a/hubblestack_pulsar_config.yaml b/hubblestack_pulsar_config.yaml new file mode 100644 index 0000000..87a4dda --- /dev/null +++ b/hubblestack_pulsar_config.yaml @@ -0,0 +1,48 @@ +/lib: { recurse: True, auto_add: True } +/bin: { recurse: True, auto_add: True } +/sbin: { recurse: True, auto_add: True } +/boot: { recurse: True, auto_add: True } +/lib64: { recurse: True, auto_add: True } +/usr/lib: { recurse: True, auto_add: True } +/usr/bin: { recurse: True, auto_add: True } +/usr/sbin: { recurse: True, auto_add: True } +/usr/lib64: { recurse: True, auto_add: True } +/usr/local/etc: { recurse: True, auto_add: True } +/usr/local/bin: { recurse: True, auto_add: True } +/usr/local/lib: { recurse: True, auto_add: True } +/usr/local/sbin: { recurse: True, auto_add: True } +/etc: + exclude: + - /etc/passwd.lock + - /etc/shadow.lock + - /etc/gshadow.lock + - /etc/group.lock + - /etc/passwd+ + - /etc/passwd- + - /etc/shadow+ + - /etc/shadow- + - /etc/group+ + - /etc/group- + - /etc/gshadow+ + - /etc/gshadow- + - /etc/cas/timestamp + - /etc/pki/nssdb/key4.db-journal + - /etc/pki/nssdb/cert9.db-journal + - /etc/salt/gpgkeys/random_seed + recurse: True + auto_add: True +/var: + exclude: + - /var/log + - /var/spool + - /var/cache + - /var/lock + - /var/lib/ntp + - /var/lib/mlocate + - /var/lib/logrotate.status + recurse: True + audo_add: True +return: slack_pulsar +checksum: sha256 +stats: True +batch: True diff --git a/pillar.example b/pillar.example index b0d5809..35f7eb1 100644 --- a/pillar.example +++ b/pillar.example @@ -1,50 +1,4 @@ beacons: pulsar: - /lib: { recurse: True, auto_add: True } - /bin: { recurse: True, auto_add: True } - /sbin: { recurse: True, auto_add: True } - /boot: { recurse: True, auto_add: True } - /lib64: { recurse: True, auto_add: True } - /usr/lib: { recurse: True, auto_add: True } - /usr/bin: { recurse: True, auto_add: True } - /usr/sbin: { recurse: True, auto_add: True } - /usr/lib64: { recurse: True, auto_add: True } - /usr/local/etc: { recurse: True, auto_add: True } - /usr/local/bin: { recurse: True, auto_add: True } - /usr/local/lib: { recurse: True, auto_add: True } - /usr/local/sbin: { recurse: True, auto_add: True } - /etc: - exclude: - - /etc/passwd.lock - - /etc/shadow.lock - - /etc/gshadow.lock - - /etc/group.lock - - /etc/passwd+ - - /etc/passwd- - - /etc/shadow+ - - /etc/shadow- - - /etc/group+ - - /etc/group- - - /etc/gshadow+ - - /etc/gshadow- - - /etc/cas/timestamp - - /etc/pki/nssdb/key4.db-journal - - /etc/pki/nssdb/cert9.db-journal - - /etc/salt/gpgkeys/random_seed - recurse: True - auto_add: True - /var: - exclude: - - /var/log - - /var/spool - - /var/cache - - /var/lock - - /var/lib/ntp - - /var/lib/mlocate - - /var/lib/logrotate.status - recurse: True - audo_add: True - return: slack_pulsar - checksum: sha256 - stats: True - batch: True + paths: + - salt://hubblestack_pulsar_config.yaml diff --git a/pulsar/_modules/checksum.py b/pulsar/_modules/checksum.py deleted file mode 100644 index 473d2c0..0000000 --- a/pulsar/_modules/checksum.py +++ /dev/null @@ -1,144 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Generic hashing script. Supports files & directories -''' -from __future__ import absolute_import - -import os -import json -import shutil -import difflib -import logging -from time import strftime - -LOG = logging.getLogger(__name__) - -__virtualname__ = 'fim' - - -def __virtual__(): - if ('file.get_hash' and 'file.stats') in __salt__: - return __virtualname__ - - -def _hasher(algo, target): - ''' - Convenience function to handle hashing - ''' - return __salt__['file.get_hash'](target, algo) - - -def _stats(target): - ''' - Convenience function to handle stats - ''' - return __salt__['file.stats'](target) - - -def checksum(algo='sha256', targets=[], excludes=[], filename='', *args, **kwargs): - ''' - Generate dictionary of hashes and corresponding filenames. - - Supports file paths and or directories. - ''' - checksums = {} - hostname = __salt__['grains.get']('fqdn') - - ## check for preconfigured algos - if not algo: - try: - if __salt__['config.get']('fim:algo'): - algo = __salt__['config.get']('fim:algo') - except KeyError: - LOG.debug('No algorithm defined. Defaulting to sha256') - - ## check for preconfigured targets - if not targets: - try: - if __salt__['config.get']('fim:targets'): - targets = __salt__['config.get']('fim:targets') - except KeyError: - return 'No targets defined. Exiting' - - ## check for preconfigured exclusions. this can be a file or a directory - if not excludes: - try: - if __salt__['config.get']('fim:exclude'): - excludes = __salt__['config.get']('fim:excludes') - except KeyError: - LOG.debug('No targets to exclude. Defaulting to empty list') - - ## iterate through list of targets and generate checksums - for target in targets: - if os.path.isdir(target): - for root, dirs, files in os.walk(target): - dirs[:] = [d for d in dirs if d not in excludes] - for file_ in files: - target = os.path.join(root, file_) - if os.path.isfile(target) and target not in excludes: - checksums[target] = {'stats': {}} - checksums[target]['stats'] = _stats(target) - checksums[target]['stats'].update({'checksum': _hasher(algo, target)}) - checksums[target]['stats'].update({'hostname': hostname}) - elif os.path.isfile(target) and target not in excludes: - checksums[target] = {'stats': {}} - checksums[target]['stats'] = _stats(target) - checksums[target]['stats'].update({'checksum': _hasher(algo, target)}) - checksums[target]['stats'].update({'hostname': hostname}) - - return checksums - - -def diff(): - ''' - Generate unified diff of two most recent fim.dat files - ''' - diff = [] - - timestamp = strftime("%Y-%m-%d") - - new_path = __salt__['config.get']('fim:new_path') - old_path = __salt__['config.get']('fim:old_path') - - root_dir = '/var/cache/salt/master/minions/' - - for minion in os.listdir(root_dir): - try: - for line in difflib.unified_diff(open(root_dir + minion + '/' + 'files' + old_path).readlines(), - open(root_dir + minion + '/' + 'files' + new_path).readlines(), n=0): - - prefix_line = False - for prefix in ('---', '+++', '@@'): - if line.startswith(prefix): - prefix_line = True - if not prefix_line: - line = line.strip() - char = line[0] - line = line[1:] - line = line.replace("'", "\"") - line = json.loads(line) - line['diff'] = char - line['timestamp'] = timestamp - diff.append(line) - except IOError: - LOG.error('No previous run to compare. Run fim.rotate.') - - ret = '\n'.join([json.dumps(s) for s in diff]) - return ret - - -def rotate(): - ''' - Rotate the fim data files to .old - ''' - new_path = __salt__['config.get']('fim:new_path') - old_path = __salt__['config.get']('fim:old_path') - - root_dir = '/var/cache/salt/master/minions/' - - for minion in os.listdir(root_dir): - shutil.copy(root_dir + minion + '/' + 'files' + new_path, - root_dir + minion + '/' + 'files' + old_path) - - return 'FIM data rotated.' - diff --git a/pulsar/inotify/etc/salt/master.d/reactor.conf b/pulsar/inotify/etc/salt/master.d/reactor.conf deleted file mode 100644 index 5de9a45..0000000 --- a/pulsar/inotify/etc/salt/master.d/reactor.conf +++ /dev/null @@ -1,3 +0,0 @@ -reactor: - - 'salt/beacon/*/inotify/*': - - /srv/reactor/slack.sls diff --git a/pulsar/inotify/pillar/beacons.sls b/pulsar/inotify/pillar/beacons.sls deleted file mode 100644 index bcc3336..0000000 --- a/pulsar/inotify/pillar/beacons.sls +++ /dev/null @@ -1,31 +0,0 @@ -beacons: - pulsar: - /etc: { recurse: True, auto_add: True } - /lib: { recurse: True, auto_add: True } - /bin: { recurse: True, auto_add: True } - /sbin: { recurse: True, auto_add: True } - /boot: { recurse: True, auto_add: True } - /lib64: { recurse: True, auto_add: True } - /usr/lib: { recurse: True, auto_add: True } - /usr/bin: { recurse: True, auto_add: True } - /usr/sbin: { recurse: True, auto_add: True } - /usr/lib64: { recurse: True, auto_add: True } - /usr/local/etc: { recurse: True, auto_add: True } - /usr/local/bin: { recurse: True, auto_add: True } - /usr/local/lib: { recurse: True, auto_add: True } - /usr/local/sbin: { recurse: True, auto_add: True } - /var: - exclude: - - /var/log - - /var/spool - - /var/cache - - /var/lock - - /var/lib/ntp - - /var/lib/mlocate - - /var/lib/logrotate.status - recurse: True - audo_add: True - return: slack_pulsar - checksum: sha256 - stats: True - batch: False diff --git a/pulsar/inotify/pillar/hubblestack.sls b/pulsar/inotify/pillar/hubblestack.sls deleted file mode 100644 index 0d8533c..0000000 --- a/pulsar/inotify/pillar/hubblestack.sls +++ /dev/null @@ -1,5 +0,0 @@ -hubblestack: - nova: - verbose: False - show_success: False - show_compliance: True diff --git a/pulsar/inotify/pillar/top.sls b/pulsar/inotify/pillar/top.sls deleted file mode 100644 index a870e4c..0000000 --- a/pulsar/inotify/pillar/top.sls +++ /dev/null @@ -1,5 +0,0 @@ -base: - '*': - - slack - - beacons - - hubblestack diff --git a/pulsar/inotify/reactor/slack.sls b/pulsar/inotify/reactor/slack.sls deleted file mode 100644 index 0974b93..0000000 --- a/pulsar/inotify/reactor/slack.sls +++ /dev/null @@ -1,32 +0,0 @@ -{% if 'IN_CREATE' in data['data']['change'] %} -slack_alert: - local.slack.post_message: - - tgt: {{ data['data']['id'] }} - - kwarg: - channel: "#channel" - message: "file: `{{ data['data']['path'] }}` CREATED on `{{ data['data']['id'] }}`." - from_name: "bot_user" - api_key: xxxx-xxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx -{% endif %} - -{% if 'IN_MODIFY' in data['data']['change'] %} -slack_alert: - local.slack.post_message: - - tgt: {{ data['data']['id'] }} - - kwarg: - channel: "#channel" - message: "file: `{{ data['data']['path'] }}` MODIFIED on `{{ data['data']['id'] }}`." - from_name: "bot_user" - api_key: xxxx-xxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx -{% endif %} - -{% if 'IN_DELETE' in data['data']['change'] %} -slack_alert: - local.slack.post_message: - - tgt: {{ data['data']['id'] }} - - kwarg: - channel: "#channel" - message: "file: `{{ data['data']['path'] }}` DELETED on `{{ data['data']['id'] }}`." - from_name: "bot_user" - api_key: xxxx-xxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx -{% endif %} diff --git a/pulsar/win_notify/pillar/top.sls b/pulsar/win_notify/pillar/top.sls deleted file mode 100644 index ccb2138..0000000 --- a/pulsar/win_notify/pillar/top.sls +++ /dev/null @@ -1,3 +0,0 @@ -base: - '*': - - beacons diff --git a/pulsar/win_notify/pillar/beacons.sls b/win_pillar.example similarity index 100% rename from pulsar/win_notify/pillar/beacons.sls rename to win_pillar.example