Merge pull request #1087 from goravsingal/fips

FIPS changes with Python 3.9

.pipeline

@@ -1,5 +1,5 @@
 
-def imgname = 'hubblestack/jenkins:centos-v1.0.17'
+def imgname = 'hubblestack/jenkins:centos-v1.0.18'
 
 pipeline {
     agent {
@@ -17,7 +17,7 @@ pipeline {
     environment {
         PY_COLORS = 1
         HS_PROFILE = 1
-        TEST_PY_V = '3.7.9'
+        TEST_PY_V = '3.9.2'
         OUTPUT = 'tests/unittests/output'
         REF = 'tests/unittests/output/relevant-files.txt'
         SKIP_0_POOL_NTP_TEST = 1

.travis.yml

@@ -16,7 +16,7 @@ env:
 
 python:
   - "3.6.10"
-  - "3.7.9"
+  - "3.9.2"
   - "3.8"
 
 install:

hubblestack/daemon.py

@@ -460,6 +460,8 @@ def load_config(args=None):
     # it will default to a platform specific file (see get_config() and DEFAULT_OPTS in hs.config)
     __opts__ = hubblestack.config.get_config(parsed_args.get('configfile'))
 
+    # Configure FIPS mode
+    setup_fips_mode()
 
     # Loading default included config options and updating them in the main __opts__
     default_include_config_options = hubblestack.config.include_config(
@@ -820,6 +822,28 @@ def clear_selective_context():
     # clear the package list so that pkg module can fetch it as fresh in next cycle
     __context__.pop('pkg.list_pkgs', None)
 
+def setup_fips_mode():
+    """
+    Setup FIPS mode
+    """
+    global __opts__
+    fips_mode_enable = __opts__.get('fips_mode', False)
+
+    if not fips_mode_enable:
+        # print('FIPS mode not configured')
+        return
+
+    if hubblestack.utils.platform.is_windows():
+        # On windows, we have to set an environment variable
+        # As Python patch is different on Windows
+        import os
+        os.environ["ENABLE_FIPS"] = "1"
+        # print('FIPS mode enabled')
+
+    import ssl
+    ssl.FIPS_mode_set(1)
+    # print('FIPS mode enabled')
+
 def parse_args(args=None):
     """
     Parse command line arguments

hubblestack/fileclient.py

@@ -900,7 +900,7 @@ def get_file(self,
                         # Master has prompted a file verification, if the
                         # verification fails, re-download the file. Try 3 times
                         d_tries += 1
-                        hsum = hubblestack.utils.hashutils.get_hash(dest, hubblestack.utils.stringutils.to_str(data.get('hash_type', b'md5')))
+                        hsum = hubblestack.utils.hashutils.get_hash(dest, hubblestack.utils.stringutils.to_str(data.get('hash_type', b'sha256')))
                         if hsum != data['hsum']:
                             log.warning(
                                 'Bad download of file %s, attempt %d of 3',
@@ -1011,7 +1011,7 @@ def __hash_and_stat_file(self, path, saltenv='base'):
                 return {}, None
             else:
                 ret = {}
-                hash_type = self.opts.get('hash_type', 'md5')
+                hash_type = self.opts.get('hash_type', 'sha256')
                 ret['hsum'] = hubblestack.utils.hashutils.get_hash(path, form=hash_type)
                 ret['hash_type'] = hash_type
                 return ret

hubblestack/grains/core.py

@@ -35,26 +35,7 @@
 __proxyenabled__ = ['*']
 __FQDN__ = None
 
-# Extend the default list of supported distros. This will be used for the
-# /etc/DISTRO-release checking that is part of linux_distribution()
-from platform import _supported_dists
-_supported_dists += ('arch', 'mageia', 'meego', 'vmware', 'bluewhite64',
-                     'slamd64', 'ovs', 'system', 'mint', 'oracle', 'void')
-
-# linux_distribution deprecated in py3.7
-try:
-    from platform import linux_distribution as _deprecated_linux_distribution
-
-    def linux_distribution(**kwargs):
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            return _deprecated_linux_distribution(**kwargs)
-except ImportError:
-    from distro import linux_distribution
-
-
-import inspect
-IS_FIPS_ENABLED = True if 'usedforsecurity' in inspect.getfullargspec(hashlib.new).kwonlyargs else False
+from distro import linux_distribution
 
 import hubblestack.exceptions
 import hubblestack.log
@@ -1391,7 +1372,7 @@ def id_():
 
 # This maps (at most) the first ten characters (no spaces, lowercased) of
 # 'osfullname' to the 'os' grain that Salt traditionally uses.
-# Please see os_data() and _supported_dists.
+# Please see os_data().
 # If your system is not detecting properly it likely needs an entry here.
 _OS_NAME_MAP = {
     'redhatente': 'RedHat',
@@ -1916,11 +1897,11 @@ def os_data():
         # (though apparently it's not intelligent enough to strip quotes)
         log.trace(
             'Getting OS name, release, and codename from '
-            'platform.linux_distribution()'
+            'distro.linux_distribution()'
         )
         (osname, osrelease, oscodename) = \
             [x.strip('"').strip("'") for x in
-             linux_distribution(supported_dists=_supported_dists)]
+             linux_distribution()]
         # Try to assign these three names based on the lsb info, they tend to
         # be more accurate than what python gets from /etc/DISTRO-release.
         # It's worth noting that Ubuntu has patched their Python distribution
@@ -2753,10 +2734,7 @@ def get_server_id():
         return {}
 
     id_ = __opts__.get('id', '')
-    if IS_FIPS_ENABLED:
-        md5 = hashlib.md5(usedforsecurity=False)
-    else:
-        md5 = hashlib.md5()
+    md5 = hashlib.md5(usedforsecurity=False)
     md5.update( str(id_).encode() )
     id_hash = int( md5.hexdigest(), 16 )
 

hubblestack/hec/obj.py

@@ -17,7 +17,6 @@
 hubble_status = hubblestack.status.HubbleStatus(__name__)
 
 from . dq import DiskQueue, NoQueue, QueueCapacityError
-from inspect import getfullargspec
 from hubblestack.utils.stdrec import update_payload
 from hubblestack.utils.encoding import encode_something_to_bytes
 
@@ -27,11 +26,9 @@
 http_event_collector_debug = False
 
 # the list of collector URLs given to the HEC object
-# are hashed into an md5 string that identifies the URL set
+# are hashed into an sha256 string that identifies the URL set
 # these maximums are per URL set, not for the entire disk cache
 max_diskqueue_size  = 10 * (1024 ** 2)
-isFipsEnabled = True if 'usedforsecurity' in getfullargspec(hashlib.new).kwonlyargs else False
-
 
 def count_input(payload):
     hs_key = ':'.join(['input', payload.sourcetype])
@@ -277,14 +274,11 @@ def __init__(self, token, index, http_event_server, host='', http_event_port='80
             self.pool_manager = urllib3.PoolManager(**pm_kw)
 
         if disk_queue:
-            if isFipsEnabled:
-                md5 = hashlib.md5(usedforsecurity=False)
-            else:
-                md5 = hashlib.md5()
+            sha_hash = hashlib.sha256()
             uril = sorted([ x.uri for x in self.server_uri ])
             for u in uril:
-                md5.update(encode_something_to_bytes(u))
-            actual_disk_queue = os.path.join(disk_queue, md5.hexdigest())
+                sha_hash.update(encode_something_to_bytes(u))
+            actual_disk_queue = os.path.join(disk_queue, sha_hash.hexdigest())
             log.debug("disk_queue for %s: %s", uril, actual_disk_queue)
             self.queue = DiskQueue(actual_disk_queue, size=disk_queue_size, compression=disk_queue_compression)
         else:

hubblestack/log/setup.py

@@ -218,7 +218,7 @@ def filter_logs(opts_to_log, remove_dots=True):
     """
     filtered_conf = _remove_sensitive_info(opts_to_log, PATTERNS_TO_FILTER)
     if remove_dots:
-        for key in filtered_conf.keys():
+        for key in filtered_conf.copy().keys():
             if '.' in key:
                 filtered_conf[key.replace('.', '_')] = filtered_conf.pop(key)
     return filtered_conf

hubblestack/modules/nebula_osquery.py

@@ -37,7 +37,6 @@
 import yaml
 import zlib
 import traceback
-from inspect import getfullargspec
 
 import hubblestack.utils.files
 import hubblestack.utils.platform
@@ -55,7 +54,6 @@
 __virtualname__ = 'nebula'
 __RESULT_LOG_OFFSET__ = {}
 OSQUERYD_NEEDS_RESTART = False
-isFipsEnabled = True if 'usedforsecurity' in getfullargspec(hashlib.new).kwonlyargs else False
 
 def __virtual__():
     return __virtualname__
@@ -1235,12 +1233,9 @@ def _osqueryd_restart_required(hashfile, flagfile):
     try:
         with open(flagfile, "r") as open_file:
             file_content = open_file.read().lower().rstrip('\n\r ').strip('\n\r')
-            if isFipsEnabled:
-                hash_md5 = hashlib.md5(usedforsecurity=False)
-            else:
-                hash_md5 = hashlib.md5()
-            hash_md5.update(file_content.encode('ISO-8859-1'))
-            new_hash = hash_md5.hexdigest()
+            hash_sha256 = hashlib.sha256()
+            hash_sha256.update(file_content.encode('ISO-8859-1'))
+            new_hash = hash_sha256.hexdigest()
 
         if not os.path.isfile(hashfile):
             with open(hashfile, "w") as hfile:
@@ -1321,12 +1316,9 @@ def _restart_osqueryd(pidfile,
 
     with open(flagfile, "r") as open_file:
         file_content = open_file.read().lower().rstrip('\n\r ').strip('\n\r')
-        if isFipsEnabled:
-            hash_md5 = hashlib.md5(usedforsecurity=False)
-        else:
-            hash_md5 = hashlib.md5()
-        hash_md5.update(file_content.encode('ISO-8859-1'))
-        new_hash = hash_md5.hexdigest()
+        hash_sha256 = hashlib.sha256()
+        hash_sha256.update(file_content.encode('ISO-8859-1'))
+        new_hash = hash_sha256.hexdigest()
 
     with open(hashfile, "w") as hfile:
         hfile.write(new_hash)

hubblestack/modules/network.py

@@ -12,7 +12,6 @@
 import socket
 import time
 from multiprocessing.pool import ThreadPool
-from inspect import getfullargspec
 
 # Import hubble libs
 import hubblestack.utils.decorators.path
@@ -22,7 +21,6 @@
 from hubblestack.exceptions import CommandExecutionError
 
 log = logging.getLogger(__name__)
-isFipsEnabled = True if 'usedforsecurity' in getfullargspec(hashlib.new).kwonlyargs else False
 
 def __virtual__():
     """
@@ -1483,110 +1481,6 @@ def mod_hostname(hostname):
 
     return True
 
-
-def connect(host, port=None, **kwargs):
-    """
-    Test connectivity to a host using a particular
-    port from the minion.
-
-    .. versionadded:: 2014.7.0
-
-    CLI Example:
-
-    .. code-block:: bash
-
-        salt '*' network.connect archlinux.org 80
-
-        salt '*' network.connect archlinux.org 80 timeout=3
-
-        salt '*' network.connect archlinux.org 80 timeout=3 family=ipv4
-
-        salt '*' network.connect google-public-dns-a.google.com port=53 proto=udp timeout=3
-    """
-
-    ret = {"result": None, "comment": ""}
-
-    if not host:
-        ret["result"] = False
-        ret["comment"] = "Required argument, host, is missing."
-        return ret
-
-    if not port:
-        ret["result"] = False
-        ret["comment"] = "Required argument, port, is missing."
-        return ret
-
-    proto = kwargs.get("proto", "tcp")
-    timeout = kwargs.get("timeout", 5)
-    family = kwargs.get("family", None)
-
-    if hubblestack.utils.validate.net.ipv4_addr(host) or hubblestack.utils.validate.net.ipv6_addr(
-        host
-    ):
-        address = host
-    else:
-        address = "{0}".format(__utils__["network.sanitize_host"](host))
-
-    try:
-        if proto == "udp":
-            __proto = socket.SOL_UDP
-        else:
-            __proto = socket.SOL_TCP
-            proto = "tcp"
-
-        if family:
-            if family == "ipv4":
-                __family = socket.AF_INET
-            elif family == "ipv6":
-                __family = socket.AF_INET6
-            else:
-                __family = 0
-        else:
-            __family = 0
-
-        (family, socktype, _proto, garbage, _address) = socket.getaddrinfo(
-            address, port, __family, 0, __proto
-        )[0]
-    except socket.gaierror:
-        ret["result"] = False
-        ret["comment"] = "Unable to resolve host {0} on {1} port {2}".format(
-            host, proto, port
-        )
-        return ret
-
-    try:
-        skt = socket.socket(family, socktype, _proto)
-        skt.settimeout(timeout)
-
-        if proto == "udp":
-            # Generate a random string of a
-            # decent size to test UDP connection
-            if isFipsEnabled:
-                md5h = hashlib.md5(usedforsecurity=False)
-            else:
-                md5h = hashlib.md5()
-            md5h.update(datetime.datetime.now().strftime("%s"))
-            msg = md5h.hexdigest()
-            skt.sendto(msg, _address)
-            recv, svr = skt.recvfrom(255)
-            skt.close()
-        else:
-            skt.connect(_address)
-            skt.shutdown(2)
-    except Exception as exc:  # pylint: disable=broad-except
-        ret["result"] = False
-        ret["comment"] = "Unable to connect to {0} ({1}) on {2} port {3}".format(
-            host, _address[0], proto, port
-        )
-        return ret
-
-    ret["result"] = True
-    ret["comment"] = "Successfully connected to {0} ({1}) on {2} port {3}".format(
-        host, _address[0], proto, port
-    )
-    return ret
-
-
 def is_private(ip_addr):
     """
     Check if the given IP address is a private address

hubblestack/utils/_compat.py

@@ -130,6 +130,9 @@ def __init__(self, address):
         else:
             self.__scope = None
 
+        # For compatibility with python3.9 ipaddress
+        self._scope_id = self.__scope
+
         # Python 3.4 fix. Versions higher are simply not affected
         # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544
         self._version = 6

hubblestack/utils/gitfs.py

@@ -425,7 +425,7 @@ def __init__(self, opts, remote, per_remote_defaults, per_remote_only,
             )
             failhard(self.role)
 
-        hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5'))
+        hash_type = getattr(hashlib, self.opts.get('hash_type', 'sha256'))
         # We loaded this data from yaml configuration files, so, its safe
         # to use UTF-8
         self.hash = hash_type(self.id.encode('utf-8')).hexdigest()

hubblestack/utils/hashutils.py

@@ -24,7 +24,7 @@ def get_hash(path, form="sha256", chunk_size=65536):
         raise ValueError('Invalid hash type: {0}'.format(form))
 
     with hubblestack.utils.files.fopen(path, 'rb') as ifile:
-        hash_obj = hash_type()
+        hash_obj = hash_type(usedforsecurity=False)    
         # read the file in in chunks, not the entire file
         for chunk in iter(lambda: ifile.read(chunk_size), b''):
             hash_obj.update(chunk)