Merge pull request #463 from hubblestack/develop

Merge to master (prep v2.4.5)

doc/conf.py

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'2.4.4'
+version = u'2.4.5'
 # The full version, including alpha/beta/rc tags.
-release = u'2.4.4-1'
+release = u'2.4.5-1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.4.4'
+__version__ = '2.4.5'

hubblestack/hangtime/__init__.py

@@ -0,0 +1,11 @@
+try:
+    import signal
+
+    # windows (et al?) has no concept of signal.SIGALRM force the issue here
+    # and, if applicable, load a fake timer wrapper
+
+    assert signal.SIGALRM > 0
+    from linux_itimers import HangTime, hangtime_wrapper
+
+except:
+    from fake import HangTime, hangtime_wrapper

hubblestack/hangtime/fake.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+'''
+Defanged fake timer setup that pretends to do all the things HangTime would do
+(by arguments); but actually does nothing at all.
+
+The decorator in particular doesn't even attempt to load the HangTime wrapper
+around code.
+'''
+
+class HangTime(object):
+    def __init__(self, msg="hang timeout detected", timeout=300, tag=None, repeats=False, decay=1.0):
+        pass
+
+    def __repr__(self):
+        return "FakeHT({:0.2f}s, tag={})".format(self.timeout, self.tag)
+
+    def restore(self, ended=False):
+        pass
+
+    def fire_timer(self, *sig_param):
+        pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, e_type, e_obj, e_tb):
+        pass
+
+
+def hangtime_wrapper(**ht_kw):
+    def _decorator(actual):
+        return actual
+    return _decorator

pkg/amazonlinux2016.09/Dockerfile

@@ -125,8 +125,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src

pkg/centos6/Dockerfile

@@ -127,8 +127,8 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src

pkg/centos7/Dockerfile

@@ -124,8 +124,8 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH=/hubble_src

pkg/coreos/Dockerfile

@@ -136,9 +136,9 @@ RUN pip -v install -r pyinstaller-requirements.txt
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian7/Dockerfile

@@ -163,9 +163,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian8/Dockerfile

@@ -145,9 +145,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/debian9/Dockerfile

@@ -140,9 +140,9 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #pyinstaller start
 #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built
 #use the following variables to choose the version of hubble
-ENV HUBBLE_CHECKOUT=v2.4.4
+ENV HUBBLE_CHECKOUT=v2.4.5
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4
+ENV HUBBLE_VERSION=2.4.5
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/amazonlinux2016.09/Dockerfile

@@ -128,7 +128,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/centos6/Dockerfile

@@ -130,7 +130,7 @@ RUN yum install -y rpmbuild rpm-build gcc make rh-ruby23 rh-ruby23-ruby-devel \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/centos7/Dockerfile

@@ -127,7 +127,7 @@ RUN yum install -y ruby ruby-devel rpmbuild rpm-build rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/coreos/Dockerfile

@@ -139,7 +139,7 @@ RUN pip -v install -r pyinstaller-requirements.txt
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/debian7/Dockerfile

@@ -166,7 +166,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/debian8/Dockerfile

@@ -148,7 +148,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/dev/debian9/Dockerfile

@@ -143,7 +143,7 @@ RUN apt-get install -y ruby ruby-dev rubygems gcc make \
 #use the following variables to choose the version of hubble
 ARG HUBBLE_CHECKOUT=develop
 ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
-ENV HUBBLE_VERSION=2.4.4_develop
+ENV HUBBLE_VERSION=2.4.5_develop
 ENV HUBBLE_ITERATION=1
 ENV HUBBLE_SRC_PATH=/hubble_src
 ENV _HOOK_DIR="./pkg/"

pkg/windows/dockerfile

@@ -3,8 +3,10 @@
 # To build an image: 1. copy pkg/windows/pyinstaller-requirements.txt &  to directory with this Dockerfile
 #                    2. docker build -t <image_name> .
 # The resulting image is ready to run the pyinstaller on container start and drop hubble<version>.exe
-# in a local directory. Mount /data volume into a directory on the host to access the package.
-# To run the container:  docker run -it --rm -v <host folder>:c:\data <image_name>
+# in a local directory. Mount c:\data volume into a directory on the host to access the package.
+# To run the container:  
+#                    3. Copy over any other items you want to include with hubble and place them in <host folder>/opt
+#                    4. docker run -it --rm -v <host folder>:c:\data <image_name>
 #build docker image from windowscore
 FROM microsoft/windowsservercore
 #Needed to just work
@@ -13,9 +15,9 @@ ENV CHOCO_URL=https://chocolatey.org/install.ps1
 #All the variables used for salt
 ENV SALT_SRC_PATH='C:/temp/salt/'
 ENV SALT_GIT_URL=https://github.com/saltstack/salt
-ENV SALT_CHECKOUT=v2018.3.0
+ENV SALT_CHECKOUT=v2018.11
 #All the variables used for hubble
-ENV HUBBLE_CHECKOUT=v2.2.4-2
+ENV HUBBLE_CHECKOUT=v2.4.5
 ENV HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git
 ENV HUBBLE_SRC_PATH='C:/temp/hubble/'
 ENV _HOOK_DIR='./pkg/'
@@ -54,7 +56,7 @@ RUN powershell.exe -Command pip install -r pyinstaller-requirements.txt;
 
 #Move portable git to a new location
 RUN powershell.exe -Command New-Item C:/temp/hubble/PortableGit -ItemType Directory; \
-  Copy-Item -Path C:/tools/git -Destination C:/temp/hubble/PortableGit -Recurse;
+  Copy-Item -Path C:/tools/git/* -Destination C:/temp/hubble/PortableGit/ -Recurse;
 
 # Modify gitfs fix for incorrect path variables until fix has been upstreamed
 RUN powershell.exe -Command If (!(Test-Path C:/Python27/Lib/site-packages/salt)) {Copy-Item C:/temp/salt/salt -Destination C:/Python27/Lib/site-packages/ -Recurse -Force}; \
@@ -78,8 +80,13 @@ CMD powershell.exe -Command Push-Location C:/temp/hubble; \
   Move-Item hubble.conf -Destination ./hubble/dist/hubble/etc/hubble/; \
   Move-Item 'hubble/PortableGit' -Destination './hubble/dist/hubble/' -Force; \
   Move-Item 'C:/ProgramData/chocolatey/lib/NSSM/tools/nssm.exe' -Destination './hubble/dist/hubble/' -Force; \
-  Move-Item 'C:/ProgramData/osquery/osqueryi.exe' -Destination './hubble/pkg/' -Force; \
+  Move-Item 'C:/ProgramData/osquery/osqueryi.exe' -Destination './hubble/dist/hubble/' -Force; \
+  If (Test-Path C:/data/hubble.conf) {Copy-Item  C:/data/hubble.conf -Destination ./hubble/dist/hubble/etc/hubble/ -Force}; \
+  If (Test-Path C:/data/opt) {Copy-Item  C:/data/opt -Destination './hubble/dist/hubble/' -Recurse -Force}; \
 #Build the installer
   Push-Location 'C:/Program Files (x86)/NSIS'; \
   ./makensis.exe /DHubbleVersion="$env:HUBBLE_CHECKOUT" 'C:/temp/hubble/pkg/windows/hubble-Setup.nsi'; \
-  Copy-Item C:/temp/hubble/pkg/windows/Hubble*exe -Destination C:/data/
+  Get-FileHash -Path C:/temp/hubble/pkg/windows/Hubble*exe -Algorithm SHA256 ^| Out-File C:/temp/hubble/pkg/windows/hubble_windows.sha256; \
+  Copy-Item C:/temp/hubble/pkg/windows/Hubble*exe -Destination C:/data/; \
+  Copy-Item C:/temp/hubble/pkg/windows/hubble_windows.sha256 -Destination C:/data/; 
+