From 921d9eb5d016caee0b1a2cc73eac5e4c21b8ab03 Mon Sep 17 00:00:00 2001 From: Samuel Felton Date: Wed, 25 Sep 2024 13:10:26 +0200 Subject: [PATCH 1/3] Set input encoding to UTF-8 to fix parsing issues --- modules/python/generator/visp_python_bindgen/header.py | 4 ++-- modules/python/generator/visp_python_bindgen/submodule.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/python/generator/visp_python_bindgen/header.py b/modules/python/generator/visp_python_bindgen/header.py index b0617e07ae..0bd3ab3ce9 100644 --- a/modules/python/generator/visp_python_bindgen/header.py +++ b/modules/python/generator/visp_python_bindgen/header.py @@ -143,7 +143,7 @@ def run_preprocessor(self): tmp_file_content.append(f'#include <{include}>\n') # Remove all includes: we only include configuration headers, defined above - with open(self.path.absolute(), 'r') as input_header_file: + with open(self.path.absolute(), 'r', encoding='utf-8') as input_header_file: include_regex = '#include\s*<(.*)>' for line in input_header_file.readlines(): matches = re.search(include_regex, line) @@ -167,7 +167,7 @@ def run_preprocessor(self): preprocessed_header_content = None # Remove all #defines that could have been left by the preprocessor - with open(preprocessor_output_path, 'r') as header_file: + with open(preprocessor_output_path, 'r', encoding='utf-8') as header_file: preprocessed_header_lines = [] for line in header_file.readlines(): if not line.startswith('#define'): diff --git a/modules/python/generator/visp_python_bindgen/submodule.py b/modules/python/generator/visp_python_bindgen/submodule.py index 5139aaa9de..67146fad32 100644 --- a/modules/python/generator/visp_python_bindgen/submodule.py +++ b/modules/python/generator/visp_python_bindgen/submodule.py @@ -85,14 +85,14 @@ def _get_config_file_or_create_default(self, path: Path) -> Dict: json.dump(default_config, config_file) return default_config else: - with open(path, 'r') as config_file: + with open(path, 'r', encoding='utf-8') as config_file: config = json.load(config_file) for included_config_filename in config.get('config_includes', []): included_config_path: Path = path.parent / included_config_filename if not included_config_path.exists(): raise RuntimeError(f'Sub config file {included_config_path} does not exist') logging.info(f'Trying to load subconfig file: {included_config_path}') - with open(included_config_path, 'r') as additional_config_file: + with open(included_config_path, 'r', encoding='utf-8') as additional_config_file: additional_config = json.load(additional_config_file) self.add_subconfig_file(config, additional_config) From 31d98b83e3526f736e57d5c1fe205d6cdd242c60 Mon Sep 17 00:00:00 2001 From: Samuel Felton Date: Thu, 26 Sep 2024 14:24:31 +0200 Subject: [PATCH 2/3] Force pcpp to use utf-8 as output --- modules/python/generator/visp_python_bindgen/generator_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/python/generator/visp_python_bindgen/generator_config.py b/modules/python/generator/visp_python_bindgen/generator_config.py index 8e9ab1e4b5..a379896a90 100644 --- a/modules/python/generator/visp_python_bindgen/generator_config.py +++ b/modules/python/generator/visp_python_bindgen/generator_config.py @@ -69,6 +69,7 @@ def to_pcpp_args_list(self) -> List[str]: args += ['-I', v] args += self.other_args args.extend(['--passthru-includes', self.passthrough_includes_regex]) + args.extend(['--output-encoding', 'utf-8']) if self.line_directive is not None: args.extend(['--line-directive', self.line_directive]) else: From c14edaece5d86ec911ad8bbf109a1b3b446d30c0 Mon Sep 17 00:00:00 2001 From: Samuel Felton Date: Thu, 26 Sep 2024 14:34:51 +0200 Subject: [PATCH 3/3] Set UTF-8 as default encoding for all file operations: see https://github.com/lagadic/visp/issues/1391#issuecomment-2376477869 --- modules/python/generator/visp_python_bindgen/generator.py | 2 +- modules/python/generator/visp_python_bindgen/header.py | 2 +- modules/python/generator/visp_python_bindgen/submodule.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/python/generator/visp_python_bindgen/generator.py b/modules/python/generator/visp_python_bindgen/generator.py index 5b80634664..a8b8aea0e5 100644 --- a/modules/python/generator/visp_python_bindgen/generator.py +++ b/modules/python/generator/visp_python_bindgen/generator.py @@ -138,7 +138,7 @@ def generate_module(generate_path: Path, config_path: Path) -> None: # Step 3: write to main.cpp the call to the submodule binding implementations. main_path = generate_path / 'main.cpp' - with open(main_path, 'w') as main_file: + with open(main_path, 'w', encoding='utf-8') as main_file: submodule_fn_declarations = [] submodule_fn_calls = [] for submodule in submodules: diff --git a/modules/python/generator/visp_python_bindgen/header.py b/modules/python/generator/visp_python_bindgen/header.py index 0bd3ab3ce9..4b7ff3c1e6 100644 --- a/modules/python/generator/visp_python_bindgen/header.py +++ b/modules/python/generator/visp_python_bindgen/header.py @@ -153,7 +153,7 @@ def run_preprocessor(self): # if 'visp3' in matches.group() or 'opencv' in matches.group(): # tmp_file_content.append(line) - with open(tmp_file_path.absolute(), 'w') as tmp_file: + with open(tmp_file_path.absolute(), 'w', encoding='utf-8') as tmp_file: tmp_file.write(''.join(tmp_file_content)) tmp_file.flush() diff --git a/modules/python/generator/visp_python_bindgen/submodule.py b/modules/python/generator/visp_python_bindgen/submodule.py index 67146fad32..5d373bc817 100644 --- a/modules/python/generator/visp_python_bindgen/submodule.py +++ b/modules/python/generator/visp_python_bindgen/submodule.py @@ -81,7 +81,7 @@ def _get_config_file_or_create_default(self, path: Path) -> Dict: 'enums': {}, 'config_includes': [] } - with open(path, 'w') as config_file: + with open(path, 'w', encoding='utf-8') as config_file: json.dump(default_config, config_file) return default_config else: @@ -195,7 +195,7 @@ def generate(self) -> None: {bindings} }} ''' - with open(self.submodule_file_path, 'w') as submodule_file: + with open(self.submodule_file_path, 'w', encoding='utf-8') as submodule_file: submodule_file.write(format_str) logs_path = self.submodule_file_path.parent.parent / 'logs'