[infra] fix windows double encoding

* force utf-8 in all open's * pump v1.4.3 - fix windows double encoding
basedosdados · Feb 24, 2021 · 8feb0eb · 8feb0eb
1 parent eb7eb6b
commit 8feb0eb
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 14 deletions.
diff --git a/basedosdados/base.py b/basedosdados/base.py
@@ -146,7 +146,7 @@ def _init_config(self, force):
 
             # Load config file
             c_file = tomlkit.parse(
-                (Path(__file__).parent / "configs/config.toml").open("r").read()
+                (Path(__file__).parent / "configs/config.toml").open("r", encoding='utf-8').read()
             )
 
             input(
@@ -248,24 +248,24 @@ def _init_config(self, force):
 
             c_file["templates_path"] = f"{Path.home()}/.basedosdados/templates"
 
-            config_file.open("w").write(tomlkit.dumps(c_file))
+            config_file.open("w", encoding='utf-8').write(tomlkit.dumps(c_file))
 
     def _load_config(self):
 
         return tomlkit.parse(
-            (Path.home() / ".basedosdados" / "config.toml").open("r").read()
+            (Path.home() / ".basedosdados" / "config.toml").open("r", encoding='utf-8').read()
         )
 
     def _load_yaml(self, file):
 
         try:
-            return yaml.load(open(file, "r"), Loader=yaml.SafeLoader)
+            return yaml.load(open(file, "r", encoding='utf-8'), Loader=yaml.SafeLoader)
         except FileNotFoundError:
             return None
 
     def _render_template(self, template_file, kargs):
 
-        return Template((self.templates / template_file).open("r").read()).render(
+        return Template((self.templates / template_file).open("r", encoding='utf-8').read()).render(
             **kargs
         )
 

diff --git a/basedosdados/dataset.py b/basedosdados/dataset.py
@@ -84,7 +84,7 @@ def init(self, replace=False):
                 )
 
                 # Write file
-                (self.dataset_folder / file.name).open("w").write(template)
+                (self.dataset_folder / file.name).open("w", encoding="utf-8").write(template)
 
         # Add code folder
         (self.dataset_folder / "code").mkdir(exist_ok=replace, parents=True)

diff --git a/basedosdados/table.py b/basedosdados/table.py
@@ -48,7 +48,6 @@ def _load_schema(self, mode="staging"):
         self._check_mode(mode)
 
         json_path = self.table_folder / f"schema-{mode}.json"
-
         columns = self.table_config["columns"]
 
         if mode == "staging":
@@ -77,8 +76,8 @@ def _load_schema(self, mode="staging"):
                         "all your column names between table_config.yaml and "
                         "publish.sql are the same?"
                     )
-
-        json.dump(columns, (json_path).open("w"))
+        ## force utf-8
+        json.dump(columns, (json_path).open("w", encoding="utf-8"))
 
         return self.client[f"bigquery_{mode}"].schema_from_json(str(json_path))
 
@@ -89,7 +88,7 @@ def _make_template(self, columns, partition_columns):
             if file.name in ["table_config.yaml", "publish.sql"]:
 
                 # Load and fill template
-                template = Template(file.open("r").read()).render(
+                template = Template(file.open("r", encoding="utf-8").read()).render(
                     bucket_name=self.bucket_name,
                     table_id=self.table_id,
                     dataset_id=self.dataset_folder.stem,
@@ -101,7 +100,9 @@ def _make_template(self, columns, partition_columns):
                 )
 
                 # Write file
-                (self.table_folder / file.name).open("w").write(template)
+                (self.table_folder / file.name).open("w", encoding="utf-8").write(
+                    template
+                )
 
     def init(
         self,
@@ -182,7 +183,7 @@ def init(
 
             if data_sample_path.suffix == ".csv":
 
-                columns = next(csv.reader(open(data_sample_path, "r")))
+                columns = next(csv.reader(open(data_sample_path, "r", encoding="utf-8")))
 
             else:
                 raise NotImplementedError(
@@ -383,6 +384,7 @@ def update(self, mode="all", not_found_ok=True):
                 / self.table_id
                 / "table_description.txt",
                 "w",
+                encoding="utf-8",
             ).write(table.description)
 
             # if m == "prod":/
@@ -421,7 +423,7 @@ def publish(self, if_exists="raise"):
             self.delete(mode="prod")
 
         self.client["bigquery_prod"].query(
-            (self.table_folder / "publish.sql").open("r").read()
+            (self.table_folder / "publish.sql").open("r", encoding="utf-8").read()
         )
 
         self.update("prod")

diff --git a/pyproject.toml b/pyproject.toml
@@ -13,7 +13,7 @@ packages = [
 ]
 readme = "README.md"
 repository = "https://github.com/base-dos-dados/bases"
-version = "1.4.2"
+version = "1.4.3"
 
 [tool.poetry.scripts]
 basedosdados = 'basedosdados.cli:cli'