Merge pull request #447 from hubblestack/develop

Merge to master (prep v2.4.2)

.gitignore

@@ -93,3 +93,5 @@ ENV/
 
 # Visual Studio Code settings
 .vscode
+
+doc/_build

README.rst

@@ -0,0 +1,8 @@
+Welcome to HubbleStack!
+=======================
+
+You can find the docs `here <https://docs.hubblestack.io>`_
+
+You can file an issue `here <https://github.com/hubblestack/hubble/issues/new>`_
+
+Follow us on `Twitter! <https://twitter.com/hubblestack>`_

doc/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = HubbleStack
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/_static/README

@@ -0,0 +1 @@
+This is a placeholder for happier version control

doc/conf.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+#
+# HubbleStack documentation build configuration file, created by
+# sphinx-quickstart on Mon Aug  6 15:46:43 2018.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.todo',
+]
+
+todo_include_todos = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'HubbleStack'
+copyright = u'2018, Colton Myers, Christer Edwards'
+author = u'Colton Myers, Christer Edwards'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u'2.4.2'
+# The full version, including alpha/beta/rc tags.
+release = u'2.4.2-1'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'README']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = ['_themes', ]
+html_theme_options = {
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+    '**': [
+        'relations.html',  # needs 'show_related': True theme option to display
+        'searchbox.html',
+    ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'HubbleStackdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'HubbleStack.tex', u'HubbleStack Documentation',
+     u'Colton Myers, Christer Edwards', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'hubblestack', u'HubbleStack Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'HubbleStack', u'HubbleStack Documentation',
+     author, 'HubbleStack', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+

doc/contents/auditing_nova.rst

@@ -0,0 +1,123 @@
+Auditing (Nova)
+===============
+
+Hubble supports success/fail auditing via a number of included modules. The
+codename for the audit piece of hubble is "Nova".
+
+Module Documentation
+--------------------
+
+:doc:`modules/nova`
+
+Usage
+-----
+
+There are two primary entry points for the Nova module:
+
+``hubble.audit``
+
+    audits the agent using the YAML profile(s) you provide as comma-separated
+    arguments.
+
+    ``hubble.audit`` takes a number of optional arguments. The first is a
+    comma-separated list of paths. These paths can be files or directories
+    within the ``hubblestack_nova_profiles`` directory, with the ``.yaml``
+    suffix removed. For information on the other arguments, please see
+    :doc:`modules/nova`.
+
+    If ``hubble.audit`` is run without targeting any audit configs or
+    directories, it will instead run ``hubble.top`` with no arguments.
+
+    ``hubble.audit`` will return a list of audits which were successful, and a
+    list of audits which failed.
+
+``hubble.top``
+
+    audits the agent using the ``top.nova`` configuration. By default, the
+    ``top.nova`` should be located in the fileserver at
+    ``salt://hubblestack_nova_profiles/top.nova``, but a different path can be
+    defined.
+
+Here are some example calls for ``hubble.audit``::
+
+    # Run the cve scanner and the CIS profile:
+    hubble hubble.audit cve.scan-v2,cis.centos-7-level-1-scored-v1
+    # Run hubble.top with the default topfile (top.nova)
+    hubble hubble.top
+    # Run all yaml configs and tags under salt://hubblestack_nova_profiles/foo/ and salt://hubblestack_nova_profiles/bar, but only run audits with tags starting with "CIS"
+    hubble hubble.audit foo,bar tags='CIS*'
+
+Configuration
+-------------
+
+For Nova module, configurations can be done via Nova topfiles. Nova topfiles
+look very similar to saltstack topfiles, except the top-level key is always
+nova, as nova doesn’t have environments.
+
+**hubblestack_data/hubblestack_nova_profiles/top.nova**::
+
+    nova:
+      '*':
+        - cve.scan-v2
+        - network.ssh
+        - network.smtp
+      'web*':
+        - cis.centos-7-level-1-scored-v1
+        - cis.centos-7-level-2-scored-v1
+      'G@os_family:debian':
+        - network.ssh
+        - cis.debian-7-level-1-scored: 'CIS*'
+
+Additionally, all nova topfile matches are compound matches, so you never need
+to define a match type like you do in saltstack topfiles. Each list item is a
+string representing the dot-separated location of a yaml file which will be run
+with ``hubble.audit``. You can also specify a tag glob to use as a filter for
+just that yaml file, using a colon after the yaml file (turning it into a
+dictionary). See the last two lines in the yaml above for examples.
+
+Examples::
+
+    hubble hubble.top
+    hubble hubble.top foo/bar/top.nova
+    hubble hubble.top foo/bar.nova verbose=True
+
+In some cases, your organization may want to skip certain audit checks for
+certain hosts. This is supported via compensating control configuration.
+
+You can skip a check globally by adding a ``control: <reason>`` key to the
+check itself. This key should be added at the same level as description and
+trigger pieces of a check. In this case, the check will never run, and will
+output under the Controlled results key.
+
+Nova also supports separate control profiles, for more fine-grained control
+using topfiles. You can use a separate YAML top-level key called control.
+Generally, you’ll put this top-level key inside of a separate YAML file and
+only include it in the top-data for the hosts for which it is relevant.
+
+For these separate control configs, the audits will always run, whether they
+are controlled or not. However, controlled audits which fail will be converted
+from Failure to Controlled in a post-processing operation.
+
+The control config syntax is as follows:
+
+**hubblestack_data/hubblestack_nova_profiles/example_control/example.yaml**::
+
+    control:
+      - CIS-2.1.4: This is the reason we control the check
+      - some_other_tag:
+          reason: This is the reason we control the check
+      - a_third_tag_with_no_reason
+
+Note that providing a reason for the control is optional. Any of the three
+formats shown in the yaml list above will work.
+
+Once you have your compensating control config, just target the yaml to the
+hosts you want to control using your topfile. In this case, all the audits will
+still run, but if any of the controlled checks fail, they will be removed from
+Failure and added to Controlled, and will be treated as a Success for the
+purposes of compliance percentage.  To use the above control, you would add the
+following to your ``top.nova`` file::
+
+    nova:
+      '*':
+        - example_control.example