kconfiglib: Update to default to UTF-8 for Python 3

Update Kconfiglib to upstream revision da40c014398f3 (+ local Zephyr
modifications) to get commit da40c014398f3 ("Force encoding to UTF-8 by
default on Python 3") in. It sets a (configurable) UTF-8 default for
Python 3, overriding the encoding specified in the current locale.

I've decided that this is a good idea after some problem reports
unrelated to Zephyr. Running with the C locale breaks things horribly
otherwise, and the fix isn't obvious.

Plain strings aren't decoded on Python 2, so no changes are needed
there.

Related PEP: https://www.python.org/dev/peps/pep-0538/

Signed-off-by: Ulf Magnusson <[email protected]>

scripts/kconfig/kconfiglib.py

@@ -514,6 +514,7 @@ class Kconfig(object):
     """
     __slots__ = (
         "_choices",
+        "_encoding",
         "_set_re_match",
         "_unset_re_match",
         "_warn_for_no_prompt",
@@ -552,7 +553,8 @@ class Kconfig(object):
     # Public interface
     #
 
-    def __init__(self, filename="Kconfig", warn=True, warn_to_stderr=True):
+    def __init__(self, filename="Kconfig", warn=True, warn_to_stderr=True,
+                 encoding="utf-8"):
         """
         Creates a new Kconfig object by parsing Kconfig files. Raises
         KconfigSyntaxError on syntax errors. Note that Kconfig files are not
@@ -589,6 +591,20 @@ def __init__(self, filename="Kconfig", warn=True, warn_to_stderr=True):
 
           This can be changed later with
           Kconfig.enable/disable_stderr_warnings().
+
+        encoding (default: "utf-8"):
+          The encoding to use when reading and writing files. If None, the
+          encoding specified in the current locale will be used.
+
+          The "utf-8" default avoids exceptions on systems that are configured
+          to use the C locale, which implies an ASCII encoding.
+
+          This parameter has no effect on Python 2, due to implementation
+          issues (regular strings turning into Unicode strings, which are
+          distinct in Python 2). Python 2 doesn't decode regular strings
+          anyway.
+
+          Related PEP: https://www.python.org/dev/peps/pep-0538/
         """
         self.srctree = os.environ.get("srctree")
 
@@ -617,6 +633,9 @@ def __init__(self, filename="Kconfig", warn=True, warn_to_stderr=True):
         self._warn_for_redun_assign = True
 
 
+        self._encoding = encoding
+
+
         self.syms = {}
         self.const_syms = {}
         self.defined_syms = []
@@ -936,7 +955,7 @@ def write_autoconf(self, filename,
           would usually want it enclosed in '/* */' to make it a C comment,
           and include a final terminating newline.
         """
-        with open(filename, "w") as f:
+        with self._open_enc(filename, "w") as f:
             f.write(header)
 
             # Avoid duplicates -- see write_config()
@@ -997,7 +1016,7 @@ def write_config(self, filename,
           would usually want each line to start with '#' to make it a comment,
           and include a final terminating newline.
         """
-        with open(filename, "w") as f:
+        with self._open_enc(filename, "w") as f:
             f.write(header)
 
             # Symbol._written is set to True when a symbol config string is
@@ -1067,7 +1086,7 @@ def write_min_config(self, filename,
           would usually want each line to start with '#' to make it a comment,
           and include a final terminating newline.
         """
-        with open(filename, "w") as f:
+        with self._open_enc(filename, "w") as f:
             f.write(header)
 
             # Avoid duplicates -- see write_config()
@@ -1229,7 +1248,7 @@ def _write_old_vals(self):
         # A separate helper function is neater than complicating write_config()
         # by passing a flag to it, plus we only need to look at symbols here.
 
-        with open("auto.conf", "w") as f:
+        with self._open_enc("auto.conf", "w") as f:
             for sym in self.defined_syms:
                 sym._written = False
 
@@ -1255,7 +1274,7 @@ def _load_old_vals(self):
             # No old values
             return
 
-        with open("auto.conf", _UNIVERSAL_NEWLINES_MODE) as f:
+        with self._open_enc("auto.conf", _UNIVERSAL_NEWLINES_MODE) as f:
             for line in f:
                 set_match = self._set_re_match(line)
                 if not set_match:
@@ -1417,12 +1436,12 @@ def _open(self, filename):
         # was set when the configuration was loaded
 
         try:
-            return open(filename, _UNIVERSAL_NEWLINES_MODE)
+            return self._open_enc(filename, _UNIVERSAL_NEWLINES_MODE)
         except IOError as e:
             if not os.path.isabs(filename) and self.srctree is not None:
                 filename = os.path.join(self.srctree, filename)
                 try:
-                    return open(filename, _UNIVERSAL_NEWLINES_MODE)
+                    return self._open_enc(filename, _UNIVERSAL_NEWLINES_MODE)
                 except IOError as e2:
                     # This is needed for Python 3, because e2 is deleted after
                     # the try block:
@@ -2577,6 +2596,14 @@ def _parse_error(self, msg):
         raise KconfigSyntaxError(
             "{}Couldn't parse '{}': {}".format(loc, self._line.rstrip(), msg))
 
+    def _open_enc(self, filename, mode):
+        # open() wrapper for forcing the encoding on Python 3. Forcing the
+        # encoding on Python 2 turns strings into Unicode strings, which gets
+        # messy. Python 2 doesn't decode regular strings anyway.
+
+        return open(filename, mode) if _IS_PY2 else \
+               open(filename, mode, encoding=self._encoding)
+
     def _warn(self, msg, filename=None, linenr=None):
         # For printing general warnings
 
@@ -4867,6 +4894,9 @@ def _warn_choice_select_imply(sym, expr, expr_type):
 # constants)
 #
 
+# Are we running on Python 2?
+_IS_PY2 = sys.version_info[0] < 3
+
 # Tokens
 (
     _T_ALLNOCONFIG_Y,
@@ -5009,7 +5039,7 @@ def _warn_choice_select_imply(sym, expr, expr_type):
 ))
 
 # Use ASCII regex matching on Python 3. It's already the default on Python 2.
-_RE_ASCII = 0 if sys.version_info[0] < 3 else re.ASCII
+_RE_ASCII = 0 if _IS_PY2 else re.ASCII
 
 # Note: This hack is no longer needed as of upstream commit c226456
 # (kconfig: warn of unhandled characters in Kconfig commands). It
@@ -5112,4 +5142,4 @@ def _warn_choice_select_imply(sym, expr, expr_type):
 #
 # There's no appreciable performance difference between "r" and "rU" for
 # parsing performance on Python 2.
-_UNIVERSAL_NEWLINES_MODE = "rU" if sys.version_info[0] < 3 else "r"
+_UNIVERSAL_NEWLINES_MODE = "rU" if _IS_PY2 else "r"