diff --git a/.pipeline b/.pipeline index 07fb7fd25..e84b4bc42 100644 --- a/.pipeline +++ b/.pipeline @@ -1,5 +1,5 @@ -def imgname = 'hubblestack/jenkins:centos-v1.0.17' +def imgname = 'hubblestack/jenkins:centos-v1.0.18' pipeline { agent { @@ -17,7 +17,7 @@ pipeline { environment { PY_COLORS = 1 HS_PROFILE = 1 - TEST_PY_V = '3.7.9' + TEST_PY_V = '3.9.2' OUTPUT = 'tests/unittests/output' REF = 'tests/unittests/output/relevant-files.txt' SKIP_0_POOL_NTP_TEST = 1 diff --git a/.travis.yml b/.travis.yml index a6d8addb4..f119191ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ env: python: - "3.6.10" - - "3.7.9" + - "3.9.2" - "3.8" install: diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index ba0b0f7de..09e6297d5 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -460,6 +460,8 @@ def load_config(args=None): # it will default to a platform specific file (see get_config() and DEFAULT_OPTS in hs.config) __opts__ = hubblestack.config.get_config(parsed_args.get('configfile')) + # Configure FIPS mode + setup_fips_mode() # Loading default included config options and updating them in the main __opts__ default_include_config_options = hubblestack.config.include_config( @@ -820,6 +822,28 @@ def clear_selective_context(): # clear the package list so that pkg module can fetch it as fresh in next cycle __context__.pop('pkg.list_pkgs', None) +def setup_fips_mode(): + """ + Setup FIPS mode + """ + global __opts__ + fips_mode_enable = __opts__.get('fips_mode', False) + + if not fips_mode_enable: + # print('FIPS mode not configured') + return + + if hubblestack.utils.platform.is_windows(): + # On windows, we have to set an environment variable + # As Python patch is different on Windows + import os + os.environ["ENABLE_FIPS"] = "1" + # print('FIPS mode enabled') + + import ssl + ssl.FIPS_mode_set(1) + # print('FIPS mode enabled') + def parse_args(args=None): """ Parse command line arguments diff --git a/hubblestack/fileclient.py b/hubblestack/fileclient.py index f02e8ca71..dba2f9acf 100644 --- a/hubblestack/fileclient.py +++ b/hubblestack/fileclient.py @@ -900,7 +900,7 @@ def get_file(self, # Master has prompted a file verification, if the # verification fails, re-download the file. Try 3 times d_tries += 1 - hsum = hubblestack.utils.hashutils.get_hash(dest, hubblestack.utils.stringutils.to_str(data.get('hash_type', b'md5'))) + hsum = hubblestack.utils.hashutils.get_hash(dest, hubblestack.utils.stringutils.to_str(data.get('hash_type', b'sha256'))) if hsum != data['hsum']: log.warning( 'Bad download of file %s, attempt %d of 3', @@ -1011,7 +1011,7 @@ def __hash_and_stat_file(self, path, saltenv='base'): return {}, None else: ret = {} - hash_type = self.opts.get('hash_type', 'md5') + hash_type = self.opts.get('hash_type', 'sha256') ret['hsum'] = hubblestack.utils.hashutils.get_hash(path, form=hash_type) ret['hash_type'] = hash_type return ret diff --git a/hubblestack/grains/core.py b/hubblestack/grains/core.py index 2e372716e..fc55c0d12 100644 --- a/hubblestack/grains/core.py +++ b/hubblestack/grains/core.py @@ -35,26 +35,7 @@ __proxyenabled__ = ['*'] __FQDN__ = None -# Extend the default list of supported distros. This will be used for the -# /etc/DISTRO-release checking that is part of linux_distribution() -from platform import _supported_dists -_supported_dists += ('arch', 'mageia', 'meego', 'vmware', 'bluewhite64', - 'slamd64', 'ovs', 'system', 'mint', 'oracle', 'void') - -# linux_distribution deprecated in py3.7 -try: - from platform import linux_distribution as _deprecated_linux_distribution - - def linux_distribution(**kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return _deprecated_linux_distribution(**kwargs) -except ImportError: - from distro import linux_distribution - - -import inspect -IS_FIPS_ENABLED = True if 'usedforsecurity' in inspect.getfullargspec(hashlib.new).kwonlyargs else False +from distro import linux_distribution import hubblestack.exceptions import hubblestack.log @@ -1391,7 +1372,7 @@ def id_(): # This maps (at most) the first ten characters (no spaces, lowercased) of # 'osfullname' to the 'os' grain that Salt traditionally uses. -# Please see os_data() and _supported_dists. +# Please see os_data(). # If your system is not detecting properly it likely needs an entry here. _OS_NAME_MAP = { 'redhatente': 'RedHat', @@ -1916,11 +1897,11 @@ def os_data(): # (though apparently it's not intelligent enough to strip quotes) log.trace( 'Getting OS name, release, and codename from ' - 'platform.linux_distribution()' + 'distro.linux_distribution()' ) (osname, osrelease, oscodename) = \ [x.strip('"').strip("'") for x in - linux_distribution(supported_dists=_supported_dists)] + linux_distribution()] # Try to assign these three names based on the lsb info, they tend to # be more accurate than what python gets from /etc/DISTRO-release. # It's worth noting that Ubuntu has patched their Python distribution @@ -2753,10 +2734,7 @@ def get_server_id(): return {} id_ = __opts__.get('id', '') - if IS_FIPS_ENABLED: - md5 = hashlib.md5(usedforsecurity=False) - else: - md5 = hashlib.md5() + md5 = hashlib.md5(usedforsecurity=False) md5.update( str(id_).encode() ) id_hash = int( md5.hexdigest(), 16 ) diff --git a/hubblestack/hec/obj.py b/hubblestack/hec/obj.py index 0f3ba2b36..996e2260f 100644 --- a/hubblestack/hec/obj.py +++ b/hubblestack/hec/obj.py @@ -17,7 +17,6 @@ hubble_status = hubblestack.status.HubbleStatus(__name__) from . dq import DiskQueue, NoQueue, QueueCapacityError -from inspect import getfullargspec from hubblestack.utils.stdrec import update_payload from hubblestack.utils.encoding import encode_something_to_bytes @@ -27,11 +26,9 @@ http_event_collector_debug = False # the list of collector URLs given to the HEC object -# are hashed into an md5 string that identifies the URL set +# are hashed into an sha256 string that identifies the URL set # these maximums are per URL set, not for the entire disk cache max_diskqueue_size = 10 * (1024 ** 2) -isFipsEnabled = True if 'usedforsecurity' in getfullargspec(hashlib.new).kwonlyargs else False - def count_input(payload): hs_key = ':'.join(['input', payload.sourcetype]) @@ -277,14 +274,11 @@ def __init__(self, token, index, http_event_server, host='', http_event_port='80 self.pool_manager = urllib3.PoolManager(**pm_kw) if disk_queue: - if isFipsEnabled: - md5 = hashlib.md5(usedforsecurity=False) - else: - md5 = hashlib.md5() + sha_hash = hashlib.sha256() uril = sorted([ x.uri for x in self.server_uri ]) for u in uril: - md5.update(encode_something_to_bytes(u)) - actual_disk_queue = os.path.join(disk_queue, md5.hexdigest()) + sha_hash.update(encode_something_to_bytes(u)) + actual_disk_queue = os.path.join(disk_queue, sha_hash.hexdigest()) log.debug("disk_queue for %s: %s", uril, actual_disk_queue) self.queue = DiskQueue(actual_disk_queue, size=disk_queue_size, compression=disk_queue_compression) else: diff --git a/hubblestack/log/setup.py b/hubblestack/log/setup.py index 6acbf587c..e9d563f05 100644 --- a/hubblestack/log/setup.py +++ b/hubblestack/log/setup.py @@ -218,7 +218,7 @@ def filter_logs(opts_to_log, remove_dots=True): """ filtered_conf = _remove_sensitive_info(opts_to_log, PATTERNS_TO_FILTER) if remove_dots: - for key in filtered_conf.keys(): + for key in filtered_conf.copy().keys(): if '.' in key: filtered_conf[key.replace('.', '_')] = filtered_conf.pop(key) return filtered_conf diff --git a/hubblestack/modules/nebula_osquery.py b/hubblestack/modules/nebula_osquery.py index 22314d0e8..eb8b70bbf 100644 --- a/hubblestack/modules/nebula_osquery.py +++ b/hubblestack/modules/nebula_osquery.py @@ -37,7 +37,6 @@ import yaml import zlib import traceback -from inspect import getfullargspec import hubblestack.utils.files import hubblestack.utils.platform @@ -55,7 +54,6 @@ __virtualname__ = 'nebula' __RESULT_LOG_OFFSET__ = {} OSQUERYD_NEEDS_RESTART = False -isFipsEnabled = True if 'usedforsecurity' in getfullargspec(hashlib.new).kwonlyargs else False def __virtual__(): return __virtualname__ @@ -1235,12 +1233,9 @@ def _osqueryd_restart_required(hashfile, flagfile): try: with open(flagfile, "r") as open_file: file_content = open_file.read().lower().rstrip('\n\r ').strip('\n\r') - if isFipsEnabled: - hash_md5 = hashlib.md5(usedforsecurity=False) - else: - hash_md5 = hashlib.md5() - hash_md5.update(file_content.encode('ISO-8859-1')) - new_hash = hash_md5.hexdigest() + hash_sha256 = hashlib.sha256() + hash_sha256.update(file_content.encode('ISO-8859-1')) + new_hash = hash_sha256.hexdigest() if not os.path.isfile(hashfile): with open(hashfile, "w") as hfile: @@ -1321,12 +1316,9 @@ def _restart_osqueryd(pidfile, with open(flagfile, "r") as open_file: file_content = open_file.read().lower().rstrip('\n\r ').strip('\n\r') - if isFipsEnabled: - hash_md5 = hashlib.md5(usedforsecurity=False) - else: - hash_md5 = hashlib.md5() - hash_md5.update(file_content.encode('ISO-8859-1')) - new_hash = hash_md5.hexdigest() + hash_sha256 = hashlib.sha256() + hash_sha256.update(file_content.encode('ISO-8859-1')) + new_hash = hash_sha256.hexdigest() with open(hashfile, "w") as hfile: hfile.write(new_hash) diff --git a/hubblestack/modules/network.py b/hubblestack/modules/network.py index d5c2ce02c..aac3b1ea0 100644 --- a/hubblestack/modules/network.py +++ b/hubblestack/modules/network.py @@ -12,7 +12,6 @@ import socket import time from multiprocessing.pool import ThreadPool -from inspect import getfullargspec # Import hubble libs import hubblestack.utils.decorators.path @@ -22,7 +21,6 @@ from hubblestack.exceptions import CommandExecutionError log = logging.getLogger(__name__) -isFipsEnabled = True if 'usedforsecurity' in getfullargspec(hashlib.new).kwonlyargs else False def __virtual__(): """ @@ -1483,110 +1481,6 @@ def mod_hostname(hostname): return True - -def connect(host, port=None, **kwargs): - """ - Test connectivity to a host using a particular - port from the minion. - - .. versionadded:: 2014.7.0 - - CLI Example: - - .. code-block:: bash - - salt '*' network.connect archlinux.org 80 - - salt '*' network.connect archlinux.org 80 timeout=3 - - salt '*' network.connect archlinux.org 80 timeout=3 family=ipv4 - - salt '*' network.connect google-public-dns-a.google.com port=53 proto=udp timeout=3 - """ - - ret = {"result": None, "comment": ""} - - if not host: - ret["result"] = False - ret["comment"] = "Required argument, host, is missing." - return ret - - if not port: - ret["result"] = False - ret["comment"] = "Required argument, port, is missing." - return ret - - proto = kwargs.get("proto", "tcp") - timeout = kwargs.get("timeout", 5) - family = kwargs.get("family", None) - - if hubblestack.utils.validate.net.ipv4_addr(host) or hubblestack.utils.validate.net.ipv6_addr( - host - ): - address = host - else: - address = "{0}".format(__utils__["network.sanitize_host"](host)) - - try: - if proto == "udp": - __proto = socket.SOL_UDP - else: - __proto = socket.SOL_TCP - proto = "tcp" - - if family: - if family == "ipv4": - __family = socket.AF_INET - elif family == "ipv6": - __family = socket.AF_INET6 - else: - __family = 0 - else: - __family = 0 - - (family, socktype, _proto, garbage, _address) = socket.getaddrinfo( - address, port, __family, 0, __proto - )[0] - except socket.gaierror: - ret["result"] = False - ret["comment"] = "Unable to resolve host {0} on {1} port {2}".format( - host, proto, port - ) - return ret - - try: - skt = socket.socket(family, socktype, _proto) - skt.settimeout(timeout) - - if proto == "udp": - # Generate a random string of a - # decent size to test UDP connection - if isFipsEnabled: - md5h = hashlib.md5(usedforsecurity=False) - else: - md5h = hashlib.md5() - md5h.update(datetime.datetime.now().strftime("%s")) - msg = md5h.hexdigest() - skt.sendto(msg, _address) - recv, svr = skt.recvfrom(255) - skt.close() - else: - skt.connect(_address) - skt.shutdown(2) - except Exception as exc: # pylint: disable=broad-except - ret["result"] = False - ret["comment"] = "Unable to connect to {0} ({1}) on {2} port {3}".format( - host, _address[0], proto, port - ) - return ret - - ret["result"] = True - ret["comment"] = "Successfully connected to {0} ({1}) on {2} port {3}".format( - host, _address[0], proto, port - ) - return ret - - def is_private(ip_addr): """ Check if the given IP address is a private address diff --git a/hubblestack/utils/_compat.py b/hubblestack/utils/_compat.py index f585ee33d..84a9a7bc2 100644 --- a/hubblestack/utils/_compat.py +++ b/hubblestack/utils/_compat.py @@ -130,6 +130,9 @@ def __init__(self, address): else: self.__scope = None + # For compatibility with python3.9 ipaddress + self._scope_id = self.__scope + # Python 3.4 fix. Versions higher are simply not affected # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544 self._version = 6 diff --git a/hubblestack/utils/gitfs.py b/hubblestack/utils/gitfs.py index 8d6f7f40b..c2d9d4f1b 100644 --- a/hubblestack/utils/gitfs.py +++ b/hubblestack/utils/gitfs.py @@ -425,7 +425,7 @@ def __init__(self, opts, remote, per_remote_defaults, per_remote_only, ) failhard(self.role) - hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5')) + hash_type = getattr(hashlib, self.opts.get('hash_type', 'sha256')) # We loaded this data from yaml configuration files, so, its safe # to use UTF-8 self.hash = hash_type(self.id.encode('utf-8')).hexdigest() diff --git a/hubblestack/utils/hashutils.py b/hubblestack/utils/hashutils.py index 0550070d9..8538fdeed 100644 --- a/hubblestack/utils/hashutils.py +++ b/hubblestack/utils/hashutils.py @@ -24,7 +24,7 @@ def get_hash(path, form="sha256", chunk_size=65536): raise ValueError('Invalid hash type: {0}'.format(form)) with hubblestack.utils.files.fopen(path, 'rb') as ifile: - hash_obj = hash_type() + hash_obj = hash_type(usedforsecurity=False) # read the file in in chunks, not the entire file for chunk in iter(lambda: ifile.read(chunk_size), b''): hash_obj.update(chunk) diff --git a/hubblestack/utils/platform.py b/hubblestack/utils/platform.py index 32fc93b03..329f77f22 100644 --- a/hubblestack/utils/platform.py +++ b/hubblestack/utils/platform.py @@ -7,16 +7,8 @@ import sys import warnings -# linux_distribution deprecated in py3.7 -try: - from platform import linux_distribution as _deprecated_linux_distribution - - def linux_distribution(**kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - return _deprecated_linux_distribution(**kwargs) -except ImportError: - from distro import linux_distribution + +from distro import linux_distribution from hubblestack.utils.decorators.memoize import memoize diff --git a/hubblestack/utils/signing.py b/hubblestack/utils/signing.py index a1e1e5528..93c88faa9 100644 --- a/hubblestack/utils/signing.py +++ b/hubblestack/utils/signing.py @@ -300,7 +300,7 @@ def __init__(self, public_crt=None, ca_crt=None, extra_crt=None): if public_crt is None: public_crt = Options.public_crt - if ca_crt is None: + if ca_crt is None or not ca_crt: ca_crt = Options.ca_crt if isinstance(ca_crt, (list, tuple)): diff --git a/pkg/abandoned/centos6/Dockerfile b/pkg/abandoned/centos6/Dockerfile index f06635d8a..990fcad0e 100644 --- a/pkg/abandoned/centos6/Dockerfile +++ b/pkg/abandoned/centos6/Dockerfile @@ -99,7 +99,7 @@ RUN yum install -y bzip2-devel sqlite-devel \ && rm -rf /var/cache/yum # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/abandoned/debian7/Dockerfile b/pkg/abandoned/debian7/Dockerfile index e2ecbdc80..0395dd49d 100644 --- a/pkg/abandoned/debian7/Dockerfile +++ b/pkg/abandoned/debian7/Dockerfile @@ -174,7 +174,7 @@ RUN mkdir -p "$LIBGIT2TEMP" \ && make install # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/usr/local/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile index 63141e18d..e9fe5543b 100644 --- a/pkg/amazonlinux2016.09/Dockerfile +++ b/pkg/amazonlinux2016.09/Dockerfile @@ -97,7 +97,7 @@ RUN yum install -y bzip2-devel sqlite-devel \ && rm -rf /var/cache/yum # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile index 6c51dfa21..11de5bc5b 100644 --- a/pkg/centos7/Dockerfile +++ b/pkg/centos7/Dockerfile @@ -96,8 +96,34 @@ RUN yum install -y bzip2-devel sqlite-devel \ && yum clean all \ && rm -rf /var/cache/yum -# use pyenv -ARG PYENV_VERSION=3.7.9 + +# OpenSSL build steps +# We are building Openssl-fips to enable FIPS mode through Python. +# This build of openssl is a specialised build for FIPS only. +# https://www.openssl.org/docs/fips.html +RUN mkdir -p /usr/local/src/ /python_installation \ + && cd /usr/local/src/ \ + && curl -O https://www.openssl.org/source/openssl-fips-2.0.16.tar.gz \ + && curl -O https://www.openssl.org/source/openssl-1.0.2t.tar.gz \ + && tar -xvf openssl-fips-2.0.16.tar.gz \ + && cd openssl-fips-2.0.16 \ + && ./config \ + && make install \ + && cd ../ \ + && tar -xvf openssl-1.0.2t.tar.gz \ + && cd openssl-1.0.2t \ + && ./config shared fips no-ssl2 no-ssl3 \ + && make depend \ + && make install \ + && cd ../ && rm -rf openssl-fips-2.0.16.tar.gz ./openssl-fips-2.0.16 \ + && echo "/usr/local/ssl/lib" > /etc/ld.so.conf.d/openssl-1.0.2t.conf \ + && ldconfig -v + +# This patch in Python is to expose get/set methods in Python. +# so that we can enable fips mode via openssl +RUN yum -y install patch +ADD python_patch_3.9.2.patch /python_installation/ +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH @@ -107,12 +133,9 @@ RUN umask 022 \ && chmod 0755 /usr/bin/pyenv-installer \ && /usr/bin/pyenv-installer \ && eval "$(pyenv init -)" \ - && pyenv install $PYENV_VERSION \ + && pyenv install --patch $PYENV_VERSION < /python_installation/python_patch_3.9.2.patch \ && pyenv global $PYENV_VERSION -RUN eval "$(pyenv init -)" \ - && pip -v install --upgrade pip - #extract osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar ADD ${OSQUERY_TAR_FILENAME} /opt/osquery/ @@ -121,7 +144,7 @@ RUN /opt/osquery/osqueryi --version #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ARG HUBBLE_CHECKOUT=v4.0.0 +ARG HUBBLE_CHECKOUT=fips ARG HUBBLE_VERSION=4.5.0 ENV HUBBLE_ITERATION=1 ENV HUBBLE_URL=https://github.com/hubblestack/hubble diff --git a/pkg/centos7/python_patch_3.9.2.patch b/pkg/centos7/python_patch_3.9.2.patch new file mode 100644 index 000000000..bf92c0d59 --- /dev/null +++ b/pkg/centos7/python_patch_3.9.2.patch @@ -0,0 +1,98 @@ +diff -aur a/Lib/ssl.py Lib/ssl.py +--- a/Lib/ssl.py 2020-10-05 15:07:58.000000000 +0000 ++++ Lib/ssl.py 2021-03-02 04:23:32.026226000 +0000 +@@ -111,6 +111,11 @@ + # LibreSSL does not provide RAND_egd + pass + ++try: ++ from _ssl import FIPS_mode, FIPS_mode_set ++except ImportError as e: ++ sys.stderr.write('error in importing\n') ++ sys.stderr.write(str(e)) + + from _ssl import ( + HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, +diff -aur a/Modules/_ssl.c Modules/_ssl.c +--- a/Modules/_ssl.c 2020-10-05 15:07:58.000000000 +0000 ++++ Modules/_ssl.c 2021-03-02 04:25:30.930669000 +0000 +@@ -5394,6 +5394,20 @@ + return PyLong_FromLong(RAND_status()); + } + ++static PyObject * ++_ssl_FIPS_mode_impl(PyObject *module) { ++ return PyLong_FromLong(FIPS_mode()); ++} ++ ++static PyObject * ++_ssl_FIPS_mode_set_impl(PyObject *module, int n) { ++ if (FIPS_mode_set(n) == 0) { ++ _setSSLError(ERR_error_string(ERR_get_error(), NULL) , 0, __FILE__, __LINE__); ++ return NULL; ++ } ++ Py_RETURN_NONE; ++} ++ + #ifndef OPENSSL_NO_EGD + /* LCOV_EXCL_START */ + /*[clinic input] +@@ -5875,6 +5889,8 @@ + _SSL_ENUM_CRLS_METHODDEF + _SSL_TXT2OBJ_METHODDEF + _SSL_NID2OBJ_METHODDEF ++ _SSL_FIPS_MODE_METHODDEF ++ _SSL_FIPS_MODE_SET_METHODDEF + {NULL, NULL} /* Sentinel */ + }; + +diff -aur a/Modules/clinic/_ssl.c.h Modules/clinic/_ssl.c.h +--- a/Modules/clinic/_ssl.c.h 2020-10-05 15:07:58.000000000 +0000 ++++ Modules/clinic/_ssl.c.h 2021-03-02 04:27:06.120295000 +0000 +@@ -1204,6 +1204,45 @@ + return _ssl_RAND_status_impl(module); + } + ++PyDoc_STRVAR(_ssl_FIPS_mode__doc__, ++"FIPS Mode"); ++ ++#define _SSL_FIPS_MODE_METHODDEF \ ++ {"FIPS_mode", (PyCFunction)_ssl_FIPS_mode, METH_NOARGS, _ssl_FIPS_mode__doc__}, ++ ++static PyObject * ++_ssl_FIPS_mode_impl(PyObject *module); ++ ++static PyObject * ++_ssl_FIPS_mode(PyObject *module, PyObject *Py_UNUSED(ignored)) ++{ ++ return _ssl_FIPS_mode_impl(module); ++} ++ ++PyDoc_STRVAR(_ssl_FIPS_mode_set_doc__, ++"FIPS Mode Set"); ++ ++#define _SSL_FIPS_MODE_SET_METHODDEF \ ++ {"FIPS_mode_set", (PyCFunction)_ssl_FIPS_mode_set, METH_O, _ssl_FIPS_mode_set_doc__}, ++ ++static PyObject * ++_ssl_FIPS_mode_set_impl(PyObject *module, int n); ++ ++static PyObject * ++_ssl_FIPS_mode_set(PyObject *module, PyObject *arg) ++{ ++ PyObject *return_value = NULL; ++ int n; ++ ++ if (!PyArg_Parse(arg, "i:FIPS_mode_set", &n)) { ++ goto exit; ++ } ++ return_value = _ssl_FIPS_mode_set_impl(module, n); ++ ++exit: ++ return return_value; ++} ++ + #if !defined(OPENSSL_NO_EGD) + + PyDoc_STRVAR(_ssl_RAND_egd__doc__, + diff --git a/pkg/centos8/Dockerfile b/pkg/centos8/Dockerfile index 2c6f6eb3e..c86b6eb22 100644 --- a/pkg/centos8/Dockerfile +++ b/pkg/centos8/Dockerfile @@ -100,7 +100,7 @@ RUN gem install --no-ri --no-rdoc ffi --version 1.12.2 \ && gem install --no-ri --no-rdoc fpm --version 1.11 # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile index 5e6f2a5d2..29e9eb78a 100644 --- a/pkg/coreos/Dockerfile +++ b/pkg/coreos/Dockerfile @@ -91,7 +91,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/debian10/Dockerfile b/pkg/debian10/Dockerfile index 021211763..4498cf9fc 100644 --- a/pkg/debian10/Dockerfile +++ b/pkg/debian10/Dockerfile @@ -99,7 +99,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile index 586863253..27fff7058 100644 --- a/pkg/debian8/Dockerfile +++ b/pkg/debian8/Dockerfile @@ -98,7 +98,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.6.10 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile index e02429150..7a413ca5a 100644 --- a/pkg/debian9/Dockerfile +++ b/pkg/debian9/Dockerfile @@ -99,7 +99,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/dev/abandoned/debian7/Dockerfile b/pkg/dev/abandoned/debian7/Dockerfile index e2ecbdc80..0395dd49d 100644 --- a/pkg/dev/abandoned/debian7/Dockerfile +++ b/pkg/dev/abandoned/debian7/Dockerfile @@ -174,7 +174,7 @@ RUN mkdir -p "$LIBGIT2TEMP" \ && make install # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/usr/local/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/dev/amazonlinux2016.09/Dockerfile b/pkg/dev/amazonlinux2016.09/Dockerfile index 63141e18d..e9fe5543b 100644 --- a/pkg/dev/amazonlinux2016.09/Dockerfile +++ b/pkg/dev/amazonlinux2016.09/Dockerfile @@ -97,7 +97,7 @@ RUN yum install -y bzip2-devel sqlite-devel \ && rm -rf /var/cache/yum # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/dev/centos7/Dockerfile b/pkg/dev/centos7/Dockerfile index 6c51dfa21..5d2a55add 100644 --- a/pkg/dev/centos7/Dockerfile +++ b/pkg/dev/centos7/Dockerfile @@ -96,8 +96,29 @@ RUN yum install -y bzip2-devel sqlite-devel \ && yum clean all \ && rm -rf /var/cache/yum -# use pyenv -ARG PYENV_VERSION=3.7.9 + +# OpenSSL build steps +RUN mkdir -p /usr/local/src/ /python_installation \ + && cd /usr/local/src/ \ + && curl -O https://www.openssl.org/source/openssl-fips-2.0.16.tar.gz \ + && curl -O https://www.openssl.org/source/openssl-1.0.2t.tar.gz \ + && tar -xvf openssl-fips-2.0.16.tar.gz \ + && cd openssl-fips-2.0.16 \ + && ./config \ + && make install \ + && cd ../ \ + && tar -xvf openssl-1.0.2t.tar.gz \ + && cd openssl-1.0.2t \ + && ./config shared fips no-ssl2 no-ssl3 \ + && make depend \ + && make install \ + && cd ../ && rm -rf openssl-fips-2.0.16.tar.gz ./openssl-fips-2.0.16 \ + && echo "/usr/local/ssl/lib" > /etc/ld.so.conf.d/openssl-1.0.2t.conf \ + && ldconfig -v + +RUN yum -y install patch +ADD python_patch_3.9.2.patch /python_installation/ +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH @@ -107,12 +128,9 @@ RUN umask 022 \ && chmod 0755 /usr/bin/pyenv-installer \ && /usr/bin/pyenv-installer \ && eval "$(pyenv init -)" \ - && pyenv install $PYENV_VERSION \ + && pyenv install --patch $PYENV_VERSION < /python_installation/python_patch_3.9.2.patch \ && pyenv global $PYENV_VERSION -RUN eval "$(pyenv init -)" \ - && pip -v install --upgrade pip - #extract osquery files. optionally pass in osquery filename with OSQUERY_TAR_FILENAME build-arg ARG OSQUERY_TAR_FILENAME=osquery_4hubble.tar ADD ${OSQUERY_TAR_FILENAME} /opt/osquery/ @@ -121,13 +139,13 @@ RUN /opt/osquery/osqueryi --version #pyinstaller start #commands specified for ENTRYPOINT and CMD are executed when the container is run, not when the image is built #use the following variables to choose the version of hubble -ARG HUBBLE_CHECKOUT=v4.0.0 +ARG HUBBLE_CHECKOUT=fips ARG HUBBLE_VERSION=4.5.0 ENV HUBBLE_ITERATION=1 -ENV HUBBLE_URL=https://github.com/hubblestack/hubble +ENV HUBBLE_URL=https://github.com/goravsingal/hubble ENV HUBBLE_DESCRIPTION="Hubble is a modular, open-source, security & compliance auditing framework which is built in python, using SaltStack as a library." ENV HUBBLE_SUMMARY="Profile based on-demand auditing and monitoring tool" -ARG HUBBLE_GIT_URL=https://github.com/hubblestack/hubble.git +ARG HUBBLE_GIT_URL=https://github.com/goravsingal/hubble.git ENV HUBBLE_SRC_PATH=/hubble_src ENV _HOOK_DIR="./pkg/" ENV _BINARY_LOG_LEVEL="INFO" diff --git a/pkg/dev/centos7/python_patch_3.9.2.patch b/pkg/dev/centos7/python_patch_3.9.2.patch new file mode 100644 index 000000000..bf92c0d59 --- /dev/null +++ b/pkg/dev/centos7/python_patch_3.9.2.patch @@ -0,0 +1,98 @@ +diff -aur a/Lib/ssl.py Lib/ssl.py +--- a/Lib/ssl.py 2020-10-05 15:07:58.000000000 +0000 ++++ Lib/ssl.py 2021-03-02 04:23:32.026226000 +0000 +@@ -111,6 +111,11 @@ + # LibreSSL does not provide RAND_egd + pass + ++try: ++ from _ssl import FIPS_mode, FIPS_mode_set ++except ImportError as e: ++ sys.stderr.write('error in importing\n') ++ sys.stderr.write(str(e)) + + from _ssl import ( + HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, +diff -aur a/Modules/_ssl.c Modules/_ssl.c +--- a/Modules/_ssl.c 2020-10-05 15:07:58.000000000 +0000 ++++ Modules/_ssl.c 2021-03-02 04:25:30.930669000 +0000 +@@ -5394,6 +5394,20 @@ + return PyLong_FromLong(RAND_status()); + } + ++static PyObject * ++_ssl_FIPS_mode_impl(PyObject *module) { ++ return PyLong_FromLong(FIPS_mode()); ++} ++ ++static PyObject * ++_ssl_FIPS_mode_set_impl(PyObject *module, int n) { ++ if (FIPS_mode_set(n) == 0) { ++ _setSSLError(ERR_error_string(ERR_get_error(), NULL) , 0, __FILE__, __LINE__); ++ return NULL; ++ } ++ Py_RETURN_NONE; ++} ++ + #ifndef OPENSSL_NO_EGD + /* LCOV_EXCL_START */ + /*[clinic input] +@@ -5875,6 +5889,8 @@ + _SSL_ENUM_CRLS_METHODDEF + _SSL_TXT2OBJ_METHODDEF + _SSL_NID2OBJ_METHODDEF ++ _SSL_FIPS_MODE_METHODDEF ++ _SSL_FIPS_MODE_SET_METHODDEF + {NULL, NULL} /* Sentinel */ + }; + +diff -aur a/Modules/clinic/_ssl.c.h Modules/clinic/_ssl.c.h +--- a/Modules/clinic/_ssl.c.h 2020-10-05 15:07:58.000000000 +0000 ++++ Modules/clinic/_ssl.c.h 2021-03-02 04:27:06.120295000 +0000 +@@ -1204,6 +1204,45 @@ + return _ssl_RAND_status_impl(module); + } + ++PyDoc_STRVAR(_ssl_FIPS_mode__doc__, ++"FIPS Mode"); ++ ++#define _SSL_FIPS_MODE_METHODDEF \ ++ {"FIPS_mode", (PyCFunction)_ssl_FIPS_mode, METH_NOARGS, _ssl_FIPS_mode__doc__}, ++ ++static PyObject * ++_ssl_FIPS_mode_impl(PyObject *module); ++ ++static PyObject * ++_ssl_FIPS_mode(PyObject *module, PyObject *Py_UNUSED(ignored)) ++{ ++ return _ssl_FIPS_mode_impl(module); ++} ++ ++PyDoc_STRVAR(_ssl_FIPS_mode_set_doc__, ++"FIPS Mode Set"); ++ ++#define _SSL_FIPS_MODE_SET_METHODDEF \ ++ {"FIPS_mode_set", (PyCFunction)_ssl_FIPS_mode_set, METH_O, _ssl_FIPS_mode_set_doc__}, ++ ++static PyObject * ++_ssl_FIPS_mode_set_impl(PyObject *module, int n); ++ ++static PyObject * ++_ssl_FIPS_mode_set(PyObject *module, PyObject *arg) ++{ ++ PyObject *return_value = NULL; ++ int n; ++ ++ if (!PyArg_Parse(arg, "i:FIPS_mode_set", &n)) { ++ goto exit; ++ } ++ return_value = _ssl_FIPS_mode_set_impl(module, n); ++ ++exit: ++ return return_value; ++} ++ + #if !defined(OPENSSL_NO_EGD) + + PyDoc_STRVAR(_ssl_RAND_egd__doc__, + diff --git a/pkg/dev/centos8/Dockerfile b/pkg/dev/centos8/Dockerfile index 2c6f6eb3e..c86b6eb22 100644 --- a/pkg/dev/centos8/Dockerfile +++ b/pkg/dev/centos8/Dockerfile @@ -100,7 +100,7 @@ RUN gem install --no-ri --no-rdoc ffi --version 1.12.2 \ && gem install --no-ri --no-rdoc fpm --version 1.11 # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/dev/coreos/Dockerfile b/pkg/dev/coreos/Dockerfile index 7d7094716..b4c78d2cf 100644 --- a/pkg/dev/coreos/Dockerfile +++ b/pkg/dev/coreos/Dockerfile @@ -91,7 +91,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/dev/debian10/Dockerfile b/pkg/dev/debian10/Dockerfile index ec971d5fe..8b67a25cf 100644 --- a/pkg/dev/debian10/Dockerfile +++ b/pkg/dev/debian10/Dockerfile @@ -99,7 +99,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/dev/debian9/Dockerfile b/pkg/dev/debian9/Dockerfile index 863684ce6..6731bfa10 100644 --- a/pkg/dev/debian9/Dockerfile +++ b/pkg/dev/debian9/Dockerfile @@ -99,7 +99,7 @@ RUN apt-get install -y libbz2-dev libsqlite3-dev \ && rm -rf /var/cache/apt # use pyenv -ARG PYENV_VERSION=3.7.9 +ARG PYENV_VERSION=3.9.2 ENV PYENV_INSTALLER_URL=https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer ENV PYENV_ROOT=/opt/hubble/pyenv ENV PATH=$PYENV_ROOT/bin:$PATH diff --git a/pkg/windows/Dockerfile b/pkg/windows/Dockerfile index 9ae8fd092..daa43f17c 100644 --- a/pkg/windows/Dockerfile +++ b/pkg/windows/Dockerfile @@ -9,57 +9,33 @@ # 4. docker run -it --rm -v :c:\data #build docker image from windowscore #the sha is used here to pin the image that has been working correctly before + FROM microsoft/windowsservercore@sha256:10e43e24be6eb5f3e19e705a88adb9794b569028f0e0d715d40f25e53e04c3fc -#Needed to just work -ENV PYTHONIOENCODING='UTF-8' -ENV _HOOK_DIR='./pkg/' -ENV NSIS_LOC='C:/Program Files (x86)/NSIS' -SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -#Create location for build environment and set as working dir -RUN New-Item C:/temp -ItemType Directory; \ - New-Item C:/data -ItemType Directory; -WORKDIR C:/temp -VOLUME C:/data + +# Download the tools +SHELL ["cmd", "/S", "/C"] +ADD "https://aka.ms/vs/15/release/vs_community.exe" "C:\TEMP\vs_community.exe" + +# Install VS 2017 community +RUN C:\TEMP\vs_community.exe --includeRecommended --includeOptional --quiet --nocache --norestart --wait \ + --add Microsoft.VisualStudio.Workload.Python \ + || IF "%ERRORLEVEL%"=="3010" EXIT 0 -#Installing Python 3.7.9 -ENV PYTHON_VERSION 3.7.9 -ENV PYTHON_RELEASE 3.7.9 +WORKDIR C:/temp +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] -RUN $url = ('https://www.python.org/ftp/python/{0}/python-{1}-amd64.exe' -f $env:PYTHON_RELEASE, $env:PYTHON_VERSION); \ - Write-Host ('Downloading {0} ...' -f $url); \ - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; \ - Invoke-WebRequest -Uri $url -OutFile 'python.exe'; \ - \ - Write-Host 'Installing ...'; \ -# https://docs.python.org/3.7/using/windows.html#installing-without-ui - Start-Process python.exe -Wait \ - -ArgumentList @( \ - '/quiet', \ - 'InstallAllUsers=1', \ - 'TargetDir=C:\Python37', \ - 'PrependPath=1', \ - 'Shortcuts=0', \ - 'Include_doc=0', \ - 'Include_pip=0', \ - 'Include_test=0' \ - ); \ - \ -#the installer updated PATH, so we should refresh our local value - $env:PATH = [Environment]::GetEnvironmentVariable('PATH', [EnvironmentVariableTarget]::Machine); \ - \ - Write-Host 'Verifying install ...'; \ - Write-Host ' python --version'; python --version; \ - \ - Write-Host 'Removing ...'; \ - Remove-Item python.exe -Force; \ - \ - Write-Host 'Complete.' +ENV CHOCO_URL=https://chocolatey.org/install.ps1 +ADD https://www.python.org/ftp/python/3.9.2/Python-3.9.2.tar.xz C:/temp/ # https://github.com/pypa/get-pip ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/d59197a3c169cef378a22428a3fa99d33e080a5d/get-pip.py ENV PYTHON_GET_PIP_SHA256 421ac1d44c0cf9730a088e337867d974b91bdce4ea2636099275071878cc189e +COPY python_patch/build.bat . +COPY python_patch/fips_python.patch . -RUN Write-Host ('Downloading get-pip.py ({0}) ...' -f $env:PYTHON_GET_PIP_URL); \ +RUN Set-ExecutionPolicy Bypass -Scope Process -Force; \ + [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls,Tls11,Tls12'; \ + Write-Host ('Downloading get-pip.py ({0}) ...' -f $env:PYTHON_GET_PIP_URL); \ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; \ Invoke-WebRequest -Uri $env:PYTHON_GET_PIP_URL -OutFile 'get-pip.py'; \ Write-Host ('Verifying sha256 ({0}) ...' -f $env:PYTHON_GET_PIP_SHA256); \ @@ -67,24 +43,34 @@ RUN Write-Host ('Downloading get-pip.py ({0}) ...' -f $env:PYTHON_GET_PIP_URL); Write-Host 'FAILED!'; \ exit 1; \ }; \ - \ - Write-Host ('Installing pip ...'); \ - python get-pip.py \ - --disable-pip-version-check \ - --no-cache-dir \ - ; \ - Remove-Item get-pip.py -Force; \ - \ - Write-Host 'Verifying pip install ...'; \ - pip --version; \ - \ - Write-Host 'Complete.' + iex ((New-Object System.Net.WebClient).DownloadString("$env:CHOCO_URL")); \ + choco install git 7zip.install patch -y; \ + 7z x Python-3.9.2.tar.xz; \ + 7z x Python-3.9.2.tar; \ + Move-Item C:\TEMP\Python-3.9.2 C:\Python39; \ + cd C:\Python39; \ + patch -p1 -i C:\temp\fips_python.patch; \ + $env:Path += ';C:\Python39\Scripts'; \ + $env:Path += ';C:\Python39'; \ + $env:Path += ';C:\Program Files\git\bin'; \ + $bkp_path = $env:Path; \ + cmd.exe /c C:\temp\build.bat; \ + Copy-Item 'C:/Python39/PCbuild/amd64/python.exe' -Destination 'C:/Python39/PCbuild/amd64/python39.exe' -Force; \ + $env:Path = $bkp_path; \ + [Environment]::SetEnvironmentVariable('PATH', $env:Path, [System.EnvironmentVariableTarget]::Machine); \ + python --version; \ + python -m pip --version; + +#Needed to just work +ENV PYTHONIOENCODING='UTF-8' +ENV _HOOK_DIR='./pkg/' +ENV NSIS_LOC='C:/Program Files (x86)/NSIS' +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] +#Create location for build environment and set as working dir + +WORKDIR C:/temp +VOLUME C:/data -##install Chocolatey, then git and osquery -ENV CHOCO_URL=https://chocolatey.org/install.ps1 -RUN Set-ExecutionPolicy Bypass -Scope Process -Force; \ - [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls,Tls11,Tls12'; \ - iex ((New-Object System.Net.WebClient).DownloadString("$env:CHOCO_URL")); RUN choco install git nssm -y; RUN choco install osquery --version 4.6.0.2 -y; diff --git a/pkg/windows/entrypoint.ps1 b/pkg/windows/entrypoint.ps1 index 8d606e4f3..a188d2bd2 100644 --- a/pkg/windows/entrypoint.ps1 +++ b/pkg/windows/entrypoint.ps1 @@ -5,7 +5,7 @@ Push-Location C:/temp/hubble; #Create pyinstaller spec and edit it to work with windows pyi-makespec --additional-hooks-dir=$env:_HOOK_DIR ./hubble.py; $specFile = Get-Content -Path ./hubble.spec; -$specFile = $specFile -replace 'a.binaries','a.binaries + [(''libeay32.dll'', ''C:\Python37\libeay32.dll'', ''BINARY'')]'; +$specFile = $specFile -replace 'a.binaries','a.binaries + [(''libeay32.dll'', ''C:\Python39\libeay32.dll'', ''BINARY'')]'; Set-Content -Path ./hubble.spec -Value $specFile -Force; pyinstaller ./hubble.spec; Pop-Location; diff --git a/pkg/windows/modules/get-settings.psm1 b/pkg/windows/modules/get-settings.psm1 index abd3e5eaa..39f326eb6 100644 --- a/pkg/windows/modules/get-settings.psm1 +++ b/pkg/windows/modules/get-settings.psm1 @@ -15,9 +15,9 @@ Function Get-Settings { # Location where the files are kept $Settings = @{ "SaltRepo" = "https://repo.saltstack.com/windows/dependencies" - "Python3Dir" = "C:\Python37" - "Scripts3Dir" = "C:\Python37\Scripts" - "SitePkgs3Dir" = "C:\Python37\Lib\site-packages" + "Python3Dir" = "C:\Python39" + "Scripts3Dir" = "C:\Python39\Scripts" + "SitePkgs3Dir" = "C:\Python39\Lib\site-packages" "DownloadDir" = "$env:Temp\DevSalt" } # The script deletes the DownLoadDir (above) for each install. diff --git a/pkg/windows/python_patch/build.bat b/pkg/windows/python_patch/build.bat new file mode 100644 index 000000000..c122a0d7a --- /dev/null +++ b/pkg/windows/python_patch/build.bat @@ -0,0 +1 @@ +"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" && C:\Python39\PCbuild\build.bat && python C:\temp\get-pip.py --disable-pip-version-check --no-cache-dir \ No newline at end of file diff --git a/pkg/windows/python_patch/fips_python.patch b/pkg/windows/python_patch/fips_python.patch new file mode 100644 index 000000000..18268dfa0 --- /dev/null +++ b/pkg/windows/python_patch/fips_python.patch @@ -0,0 +1,37 @@ +diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c +index adc8653773..19bd1867b9 100644 +--- a/Modules/_hashopenssl.c ++++ b/Modules/_hashopenssl.c +@@ -875,6 +875,13 @@ _hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, + int usedforsecurity) + /*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/ + { ++ //should work only when usedforsecurity is false (0) ++ if (getenv("ENABLE_FIPS") != NULL) { ++ if (usedforsecurity == 1) { ++ PyErr_SetString(PyExc_RuntimeError, ++ "Non fips-compliant algorithm"); ++ } ++ } + return EVP_fast_new(module, data_obj, EVP_md5(), usedforsecurity); + } + +diff --git a/Modules/md5module.c b/Modules/md5module.c +index 6ed843376a..0d3014d11b 100644 +--- a/Modules/md5module.c ++++ b/Modules/md5module.c +@@ -513,6 +513,14 @@ static PyObject * + _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) + /*[clinic end generated code: output=587071f76254a4ac input=7a144a1905636985]*/ + { ++ //should work only when usedforsecurity is false (0) ++ if (getenv("ENABLE_FIPS") != NULL) { ++ if (usedforsecurity == 1) { ++ PyErr_SetString(PyExc_RuntimeError, ++ "Non fips-compliant algorithm"); ++ } ++ } ++ + MD5object *new; + Py_buffer buf; + diff --git a/pkg/windows/salt_build_script.ps1 b/pkg/windows/salt_build_script.ps1 index 384ccd246..6ab46de3d 100644 --- a/pkg/windows/salt_build_script.ps1 +++ b/pkg/windows/salt_build_script.ps1 @@ -162,16 +162,16 @@ Write-Output " ----------------------------------------------------------------" Write-Output " - $script_name :: Installing windows specific pypi resources using pip . . ." Write-Output " ----------------------------------------------------------------" if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req_win.txt" "pip install" } else { if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) { # folder empty Write-Output " pip download from req_win.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip download" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip download" } Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE" Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_win.txt" "pip install" } #============================================================================== @@ -183,16 +183,16 @@ If ($NoPipDependencies -eq $false) { Write-Output " - $script_name :: Installing pypi resources using pip . . ." Write-Output " ----------------------------------------------------------------" if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python -m pip --disable-pip-version-check --no-cache-dir install -r $($script_path)\req.txt" "pip install" } else { if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) { # folder empty Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python -m pip --disable-pip-version-check download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download" } Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE" Write-Output " If a (new) resource is missing, please delete all files in this cache, go online and repeat" - Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python.exe -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install" + Start_Process_and_test_exitcode "cmd" "/c $($ini['Settings']['Python3Dir'])\python -m pip --disable-pip-version-check install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install" } } diff --git a/requirements.txt b/requirements.txt index 03f054183..e8135dd9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ certifi cffi chardet croniter -cryptography +# cryptography daemon distro futures diff --git a/setup.py b/setup.py index 576242b03..db93d136d 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'pyyaml', 'objgraph', 'pycryptodome', - 'cryptography', + # 'cryptography', 'pyopenssl', 'requests>=2.13.0', 'daemon', diff --git a/tests/unittests/modules/test_network_module.py b/tests/unittests/modules/test_network_module.py index 58274d327..01698f3f3 100644 --- a/tests/unittests/modules/test_network_module.py +++ b/tests/unittests/modules/test_network_module.py @@ -206,57 +206,6 @@ def test_mod_hostname(self): ): self.assertTrue(network.mod_hostname("hostname")) - def test_connect(self): - """ - Test for Test connectivity to a host using a particular - port from the minion. - """ - with patch("socket.socket") as mock_socket: - self.assertDictEqual( - network.connect(False, "port"), - {"comment": "Required argument, host, is missing.", "result": False}, - ) - - self.assertDictEqual( - network.connect("host", False), - {"comment": "Required argument, port, is missing.", "result": False}, - ) - - ret = "Unable to connect to host (0) on tcp port port" - mock_socket.side_effect = Exception("foo") - with patch.dict( - network.__utils__, - {"network.sanitize_host": MagicMock(return_value="A")}, - ): - with patch.object( - socket, - "getaddrinfo", - return_value=[["ipv4", "A", 6, "B", "0.0.0.0"]], - ): - self.assertDictEqual( - network.connect("host", "port"), - {"comment": ret, "result": False}, - ) - - ret = "Successfully connected to host (0) on tcp port port" - mock_socket.side_effect = MagicMock() - mock_socket.settimeout().return_value = None - mock_socket.connect().return_value = None - mock_socket.shutdown().return_value = None - with patch.dict( - network.__utils__, - {"network.sanitize_host": MagicMock(return_value="A")}, - ): - with patch.object( - socket, - "getaddrinfo", - return_value=[["ipv4", "A", 6, "B", "0.0.0.0"]], - ): - self.assertDictEqual( - network.connect("host", "port"), - {"comment": ret, "result": True}, - ) - @skipIf(not bool(ipaddress), "unable to import 'ipaddress'") def test_is_private(self): """