From 426b37f187f8c383c3a95092394083b7a7e6b7d5 Mon Sep 17 00:00:00 2001 From: mikecooke77 Date: Mon, 18 Nov 2024 07:48:17 +0000 Subject: [PATCH 1/4] Modify code to read spaces before direct include command --- src/yamlprocessor/datapreprocessor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/yamlprocessor/datapreprocessor.py b/src/yamlprocessor/datapreprocessor.py index 0de05af..d8c3367 100755 --- a/src/yamlprocessor/datapreprocessor.py +++ b/src/yamlprocessor/datapreprocessor.py @@ -48,6 +48,8 @@ def process_yaml(self, in_yaml, out_yaml): for iline in lines: # look for specific pattern in each line if 'DIRECT_INCLUDE=' in iline: + # get spaces before 'DIRECT_INCLUDE' + spaces = newstring.split('DIRECT_INCLUDE')[0] # retrieve header file yaml_header_File = iline.split('=')[1].rstrip() # replace variables in the string @@ -56,8 +58,10 @@ def process_yaml(self, in_yaml, out_yaml): # open header file with open(yaml_header_File, 'r') as file: auxFileData = file.read() + # add spaces to front of lines + auxFileData_offset = [spaces + line for line in auxFileData] # update lines for new file - new_line.append(auxFileData) + new_line.append(auxFileData_offset) else: new_line.append(iline) # save the result From 48fe487767fe76fa41d4c54b4a9dcbb4534b2e66 Mon Sep 17 00:00:00 2001 From: mikecooke77 Date: Mon, 18 Nov 2024 08:51:35 +0000 Subject: [PATCH 2/4] Add the ability to use the preprocessor with offset --- src/yamlprocessor/datapreprocessor.py | 21 +++++----- .../tests/test_datapreprocess.py | 42 +++++++++++++++++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/src/yamlprocessor/datapreprocessor.py b/src/yamlprocessor/datapreprocessor.py index d8c3367..5d34d90 100755 --- a/src/yamlprocessor/datapreprocessor.py +++ b/src/yamlprocessor/datapreprocessor.py @@ -49,19 +49,18 @@ def process_yaml(self, in_yaml, out_yaml): # look for specific pattern in each line if 'DIRECT_INCLUDE=' in iline: # get spaces before 'DIRECT_INCLUDE' - spaces = newstring.split('DIRECT_INCLUDE')[0] + spaces = iline.split('DIRECT_INCLUDE')[0] # retrieve header file - yaml_header_File = iline.split('=')[1].rstrip() + yaml_aux_File = iline.split('=')[1].rstrip() # replace variables in the string - yaml_header_File = self.__replace_placeholders( - yaml_header_File) - # open header file - with open(yaml_header_File, 'r') as file: - auxFileData = file.read() - # add spaces to front of lines - auxFileData_offset = [spaces + line for line in auxFileData] - # update lines for new file - new_line.append(auxFileData_offset) + yaml_aux_File = self.__replace_placeholders( + yaml_aux_File) + # open and read header file + with open(yaml_aux_File, 'r') as file: + auxFileData = file.readlines() + # Add spaces and append to output + for auxline in auxFileData: + new_line.append(spaces + auxline) else: new_line.append(iline) # save the result diff --git a/src/yamlprocessor/tests/test_datapreprocess.py b/src/yamlprocessor/tests/test_datapreprocess.py index 9bbd6c5..2037a9f 100644 --- a/src/yamlprocessor/tests/test_datapreprocess.py +++ b/src/yamlprocessor/tests/test_datapreprocess.py @@ -66,3 +66,45 @@ def test_main_0(tmp_path, yaml): outfilename1 = tmp_path / 'test_1.yaml' preprocessor.process_yaml(tmp_path / 'in_1.yaml', outfilename1) assert yaml.load(outfilename1.open()) == ref_yaml + +def test_main_1(tmp_path, yaml): + """Test direct insert with spaces.""" + yaml_in = """ +_: +- &banana 1 +- &groups [4, 5, 6] + +data: + brain: *banana + DIRECT_INCLUDE=$FILE_PATH/aux.yaml +""" + yaml_aux = """ +tel: *groups +""" + reference = """ +_: +- &banana 1 +- &groups [4, 5, 6] + +data: + brain: *banana + tel: *groups +""" + infilename = tmp_path / 'in_0.yaml' + with infilename.open('w') as infile: + infile.write(yaml_in) + + auxfilename = tmp_path / 'aux.yaml' + with auxfilename.open('w') as auxfile: + auxfile.write(yaml_aux) + + # Run preprocessor + preprocessor = DataPreProcessor() + keymap = {"FILE_PATH": str(tmp_path)} + preprocessor.add_replacements_map(keymap) + # Setup reference + ref_yaml = yaml.load(reference) + # Test first style input + outfilename0 = tmp_path / 'test_0.yaml' + preprocessor.process_yaml(tmp_path / 'in_0.yaml', outfilename0) + assert yaml.load(outfilename0.open()) == ref_yaml From 629edb806710bb5e6c7fc928bd5c9ddc02f27f55 Mon Sep 17 00:00:00 2001 From: mikecooke77 Date: Mon, 18 Nov 2024 08:58:15 +0000 Subject: [PATCH 3/4] Add a line to the documentation --- docs/data-preprocessor.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/data-preprocessor.rst b/docs/data-preprocessor.rst index b98b9cb..b6be8cd 100644 --- a/docs/data-preprocessor.rst +++ b/docs/data-preprocessor.rst @@ -3,7 +3,9 @@ Data Pre-Processor The preprocessor looks for the DIRECT_INCLUDE= keyword in the input yaml and concatenates the associated file at this point in the input file. The result -is written to the output file or standard out if - is specified. +is written to the output file or standard out if - is specified. If the +DIRECT_INCLUDE keyword has proceeding spaces these will be used to offset +the lines which are being concatenated. It is expected that the keyword in the input yaml file will take the following format: From 543e618ed34853ccec5cd544e1a235382fe612bc Mon Sep 17 00:00:00 2001 From: mikecooke77 Date: Mon, 18 Nov 2024 09:07:35 +0000 Subject: [PATCH 4/4] Add extra line for flake --- src/yamlprocessor/tests/test_datapreprocess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yamlprocessor/tests/test_datapreprocess.py b/src/yamlprocessor/tests/test_datapreprocess.py index 2037a9f..6d37b1e 100644 --- a/src/yamlprocessor/tests/test_datapreprocess.py +++ b/src/yamlprocessor/tests/test_datapreprocess.py @@ -67,6 +67,7 @@ def test_main_0(tmp_path, yaml): preprocessor.process_yaml(tmp_path / 'in_1.yaml', outfilename1) assert yaml.load(outfilename1.open()) == ref_yaml + def test_main_1(tmp_path, yaml): """Test direct insert with spaces.""" yaml_in = """